import numpy as np
import cupy as cp

from parla import Parla, parray
from parla.cuda import gpu
from parla.cpu import cpu
from parla.tasks import spawn, TaskSpace

def main():
    print("Initializing PArray")
    A = parray.asarray([[1, 2], [3, 4]])
    data = np.random.rand(2,2)
    B = parray.asarray(data)

    @spawn(placement=cpu)
    async def task_launcher():
        ts = TaskSpace("tasks")

        @spawn(ts[0], placement=gpu(0), input=[A])
        async def read_only_task():
            print(A)

        @spawn(ts[1], placement=gpu(1), output=[B])
        async def write_only_task():
            B.update(cp.sqrt(B.array))

        @spawn(ts[2], [ts[0:2]], placement=cpu, input=[A], inout=[B])
        async def write_and_write_task():
            B[0][1] = np.sum((B + A).array)

        @spawn(ts[3], [ts[2]], placement=gpu(1), inout=[A[0]])
        async def write_and_write_task():
            A[0] = cp.sqrt(A[0].array)

        await ts

if __name__ == '__main__':
    with Parla():
        main()


from array import *
class occurance_specified_element():
    def occurance(self):
        arr=array('i',[])
        arr_length = int(input("Enter the Length of the array:"))
        for x in range(arr_length):
            element=int(input("Enter the element: "))
            arr.append(element)
        print("Original array : ", arr)
        print()

        key_element=int(input("Enter the element u want to remove the first occurance: "))
        print()
        if key_element in arr:
             ("remove element is: ",arr.remove(key_element))

             print("New array: ",arr)
        else:
            print()
            print("OOPS!!it seems to be error"
                "Try again")

specified_element=occurance_specified_element()
specified_element.occurance()


from db.buckets import Buckets
from core.config import TAGS_BUCKET, ATTRIBUTE_BUCKET
from services import upsert, get, get_all, update, delete, filter_query
from services.utils import bulk_create, get_or_bulk_create, get_document_if_exists
from services.exceptions import DocumentAlreadyExists, DocumentNotFound

async def create_tag(tag):
    bucket = await Buckets.get_bucket(TAGS_BUCKET)
    tag_for_search = tag.copy()
    delattr(tag_for_search, 'attrs')
    tags = await get_document_if_exists(bucket, tag_for_search)
    if tags:
        raise DocumentAlreadyExists(tag.name)
    tag.attrs = await get_or_bulk_create(await Buckets.get_bucket(ATTRIBUTE_BUCKET), tag.attrs)
    return await upsert(bucket, tag)

async def get_all_tags(skip=0, limit=100):
    bucket = await Buckets.get_bucket(TAGS_BUCKET)
    documents = await get_all(bucket, skip=skip, limit=limit)
    return documents

async def get_tag_by_id(tag_id):
    bucket = await Buckets.get_bucket(TAGS_BUCKET)
    document = await get(bucket, tag_id)
    return document

async def update_tag_by_id(tag_id, tag):
    bucket = await Buckets.get_bucket(TAGS_BUCKET)
    if tag.attrs:
        tag.attrs = await get_or_bulk_create(await Buckets.get_bucket(ATTRIBUTE_BUCKET), tag.attrs)
    updated_tag = await update(bucket, tag, tag_id)
    if not updated_tag:
        raise DocumentNotFound(tag_id)
    return updated_tag

async def remove_tag_by_id(tag_id):
    bucket = await Buckets.get_bucket(TAGS_BUCKET)
    return await delete(bucket, tag_id)


#! /usr/bin/env python
# -*- coding: utf-8 -*-

import gzip
import json
import re

def chose_text():
    filename = "jawiki-country.json.gz"
    #reading modeでファイルを開く
    with gzip.open(filename,'rt') as open_file:
        #一行ずつjsonファイルをよんでいく
        for text in open_file:
            #json文字列を辞書に変更
            jsondata = json.loads(text)
            #titleがイギリスだったら
            if jsondata['title'] == 'イギリス':
                #本文を表示
                return jsondata['text']

# 正規表現のコンパイル
#(?:)はキャプチャ対象外
#File｜ファイル Fileかファイル
#(.+?)任意の文字1文字以上
pattern = re.compile(r'(?:File|ファイル):(.+?)\|', re.VERBOSE)

# 抽出
result = pattern.findall(chose_text())

# 結果表示
for line in result:
    print(line)


# -*- coding: utf-8 -*-
#
import os
import wx
import wx.lib.newevent
import sys

from form.panel.BasePanel import BasePanel
from form.parts.BaseFilePickerCtrl import BaseFilePickerCtrl
from form.parts.HistoryFilePickerCtrl import HistoryFilePickerCtrl
from form.parts.ConsoleCtrl import ConsoleCtrl
from form.parts.TargetBoneDialog import TargetBoneDialog
from form.worker.MultiSplitWorkerThread import MultiSplitWorkerThread
from utils import MFormUtils, MFileUtils
from utils.MLogger import MLogger # noqa

logger = MLogger(__name__)
TIMER_ID = wx.NewId()

# イベント定義
(MultiSplitThreadEvent, EVT_SMOOTH_THREAD) = wx.lib.newevent.NewEvent()

class MultiSplitPanel(BasePanel):

    def __init__(self, frame: wx.Frame, multi_split: wx.Notebook, tab_idx: int):
        super().__init__(frame, multi_split, tab_idx)
        self.timer = wx.Timer(self, TIMER_ID)
        self.convert_multi_split_worker = None

        self.header_sizer = wx.BoxSizer(wx.VERTICAL)

        self.description_txt = wx.StaticText(self, wx.ID_ANY, u"モーションの指定ボーンの移動量と回転量をXYZに分割します。分割するボーンは「ボーン指定」ボタンから定義できます。" \
                                             + "\n回転ボーンは、YXZの順番で多段化したボーンを適用すると、回転結果がオリジナルと一致します。" \
                                             + "\n不要キー削除を行うと、キーが間引きされます。キー間がオリジナルから多少ずれ、やや時間がかかります。", wx.DefaultPosition, wx.DefaultSize, 0)
        self.header_sizer.Add(self.description_txt, 0, wx.ALL, 5)

        self.static_line01 = wx.StaticLine(self, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, wx.LI_HORIZONTAL)
        self.header_sizer.Add(self.static_line01, 0, wx.EXPAND | wx.ALL, 5)

        # 対象VMDファイルコントロール
        self.vmd_file_ctrl = HistoryFilePickerCtrl(self.frame, self, u"対象モーションVMD/VPD", u"対象モーションVMDファイルを開く", ("vmd", "vpd"), wx.FLP_DEFAULT_STYLE, \
                                                   u"調整したい対象モーションのVMDパスを指定してください。\nD&Dでの指定、開くボタンからの指定、履歴からの選択ができます。", \
                                                   file_model_spacer=46, title_parts_ctrl=None, title_parts2_ctrl=None, file_histories_key="multi_split_vmd", is_change_output=True, \
                                                   is_aster=False, is_save=False, set_no=1)
        self.vmd_file_ctrl.file_ctrl.Bind(wx.EVT_FILEPICKER_CHANGED, self.on_change_file)
        self.header_sizer.Add(self.vmd_file_ctrl.sizer, 1, wx.EXPAND, 0)

        # 対象PMXファイルコントロール
        self.model_file_ctrl = HistoryFilePickerCtrl(self.frame, self, u"多段化済みモデルPMX", u"多段化済みモデルPMXファイルを開く", ("pmx"), wx.FLP_DEFAULT_STYLE, \
                                                     u"モーションを適用したいモデルのPMXパスを指定してください。\n人体モデル以外にも適用可能です。\nD&Dでの指定、開くボタンからの指定、履歴からの選択ができます。", \
                                                     file_model_spacer=49, title_parts_ctrl=None, title_parts2_ctrl=None, file_histories_key="multi_split_pmx", \
                                                     is_change_output=True, is_aster=False, is_save=False, set_no=1)
        self.model_file_ctrl.file_ctrl.Bind(wx.EVT_FILEPICKER_CHANGED, self.on_change_file)
        self.header_sizer.Add(self.model_file_ctrl.sizer, 1, wx.EXPAND, 0)

        # 出力先VMDファイルコントロール
        self.output_vmd_file_ctrl = BaseFilePickerCtrl(frame, self, u"出力対象VMD", u"出力対象VMDファイルを開く", ("vmd"), wx.FLP_OVERWRITE_PROMPT | wx.FLP_SAVE | wx.FLP_USE_TEXTCTRL, \
                                                       u"調整結果の対象VMD出力パスを指定してください。\n対象VMDファイル名に基づいて自動生成されますが、任意のパスに変更することも可能です。", \
                                                       is_aster=False, is_save=True, set_no=1)
        self.header_sizer.Add(self.output_vmd_file_ctrl.sizer, 1, wx.EXPAND, 0)

        self.sizer.Add(self.header_sizer, 0, wx.EXPAND | wx.ALL, 5)

        self.setting_sizer = wx.BoxSizer(wx.HORIZONTAL)

        # ボーン名指定
        self.bone_target_txt_ctrl = wx.TextCtrl(self, wx.ID_ANY, "", wx.DefaultPosition, (450, 50), wx.HSCROLL | wx.VSCROLL | wx.TE_MULTILINE | wx.TE_READONLY)
        self.bone_target_txt_ctrl.SetBackgroundColour(wx.SystemSettings.GetColour(wx.SYS_COLOUR_3DLIGHT))
        self.setting_sizer.Add(self.bone_target_txt_ctrl, 1, wx.EXPAND | wx.ALL, 5)

        self.bone_target_btn_ctrl = wx.Button(self, wx.ID_ANY, u"ボーン指定", wx.DefaultPosition, wx.DefaultSize, 0)
        self.bone_target_btn_ctrl.SetToolTip(u"モーションに登録されているボーンから、分割したいボーンを指定できます")
        self.bone_target_btn_ctrl.Bind(wx.EVT_BUTTON, self.on_click_bone_target)
        self.setting_sizer.Add(self.bone_target_btn_ctrl, 0, wx.ALIGN_BOTTOM | wx.ALL, 5)

        self.sizer.Add(self.setting_sizer, 0, wx.EXPAND | wx.ALL, 5)

        # 不要キー削除処理
        self.flg_sizer = wx.BoxSizer(wx.VERTICAL)
        self.remove_unnecessary_flg_ctrl = wx.CheckBox(self, wx.ID_ANY, u"不要キー削除処理を追加実行する", wx.DefaultPosition, wx.DefaultSize, 0)
        self.remove_unnecessary_flg_ctrl.SetToolTip(u"チェックを入れると、不要キー削除処理を追加で実行します。キーが減る分、キー間が少しズレる事があります。")
        self.remove_unnecessary_flg_ctrl.Bind(wx.EVT_CHECKBOX, self.on_change_file)
        self.flg_sizer.Add(self.remove_unnecessary_flg_ctrl, 0, wx.ALL, 5)
        self.sizer.Add(self.flg_sizer, 0, wx.EXPAND | wx.ALL, 5)

        btn_sizer = wx.BoxSizer(wx.HORIZONTAL)

        # 実行ボタン
        self.multi_split_btn_ctrl = wx.Button(self, wx.ID_ANY, u"多段分割", wx.DefaultPosition, wx.Size(200, 50), 0)
        self.multi_split_btn_ctrl.SetToolTip(u"キーフレを多段用に分割したモーションを生成します")
        self.multi_split_btn_ctrl.Bind(wx.EVT_LEFT_DOWN, self.on_convert_multi_split)
        self.multi_split_btn_ctrl.Bind(wx.EVT_LEFT_DCLICK, self.on_doubleclick)
        btn_sizer.Add(self.multi_split_btn_ctrl, 0, wx.ALL, 5)

        self.sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.SHAPED, 5)

        # コンソール
        self.console_ctrl = ConsoleCtrl(self, self.frame.logging_level, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.Size(-1, 420), \
                                        wx.TE_MULTILINE | wx.TE_READONLY | wx.BORDER_NONE | wx.HSCROLL | wx.VSCROLL | wx.WANTS_CHARS)
        self.console_ctrl.SetBackgroundColour(wx.SystemSettings.GetColour(wx.SYS_COLOUR_3DLIGHT))
        self.console_ctrl.Bind(wx.EVT_CHAR, lambda event: MFormUtils.on_select_all(event, self.console_ctrl))
        self.sizer.Add(self.console_ctrl, 1, wx.ALL | wx.EXPAND, 5)

        # ゲージ
        self.gauge_ctrl = wx.Gauge(self, wx.ID_ANY, 100, wx.DefaultPosition, wx.DefaultSize, wx.GA_HORIZONTAL)
        self.gauge_ctrl.SetValue(0)
        self.sizer.Add(self.gauge_ctrl, 0, wx.ALL | wx.EXPAND, 5)

        self.Layout()
        self.fit()

        # ボーン選択用ダイアログ
        self.bone_dialog = TargetBoneDialog(self.frame, self, "→", "統合")

        # フレームに変換完了処理バインド
        self.frame.Bind(EVT_SMOOTH_THREAD, self.on_convert_multi_split_result)

    def on_click_bone_target(self, event: wx.Event):
        self.disable()

        sys.stdout = self.console_ctrl
        # VMD読み込み
        self.vmd_file_ctrl.load()
        # PMX読み込み
        self.model_file_ctrl.load()

        if (self.vmd_file_ctrl.data and self.model_file_ctrl.data and \
                (self.vmd_file_ctrl.data.digest != self.bone_dialog.vmd_digest or self.model_file_ctrl.data.digest != self.bone_dialog.pmx_digest)):

            # データが揃ってたら押下可能
            self.bone_target_btn_ctrl.Enable()
            # リストクリア
            self.bone_target_txt_ctrl.SetValue("")
            # ボーン選択用ダイアログ
            self.bone_dialog.Destroy()
            self.bone_dialog = TargetBoneDialog(self.frame, self, "→", "分割")
            self.bone_dialog.initialize()
        else:
            if not self.vmd_file_ctrl.data or not self.model_file_ctrl.data:
                logger.error("対象モーションVMD/VPDもしくは多段化済みモデルPMXが未指定です。", decoration=MLogger.DECORATION_BOX)
                self.enable()
                return

        self.enable()

        if self.bone_dialog.ShowModal() == wx.ID_CANCEL:
            return     # the user changed their mind

        # 選択されたボーンリストを入力欄に設定
        bone_list = self.bone_dialog.get_bone_list()

        selections = ["{0} {7} 【回転X: {1}】【回転Y: {2}】【回転Z: {3}】【移動X: {4}】【移動Y: {5}】【移動Z: {6}】" \
                      .format(bset[0], bset[1], bset[2], bset[3], bset[4], bset[5], bset[6], "→") for bset in bone_list]
        self.bone_target_txt_ctrl.SetValue('\n'.join(selections))

        self.bone_dialog.Hide()

    # ファイル変更時の処理
    def on_change_file(self, event: wx.Event):
        self.set_output_vmd_path(event, is_force=True)

    def set_output_vmd_path(self, event: wx.Event, is_force=False):
        output_multi_split_vmd_path = MFileUtils.get_output_multi_split_vmd_path(
            self.vmd_file_ctrl.file_ctrl.GetPath(),
            self.model_file_ctrl.file_ctrl.GetPath(),
            self.output_vmd_file_ctrl.file_ctrl.GetPath(), is_force)

        self.output_vmd_file_ctrl.file_ctrl.SetPath(output_multi_split_vmd_path)

        if len(output_multi_split_vmd_path) >= 255 and os.name == "nt":
            logger.error("生成予定のファイルパスがWindowsの制限を超えています。\n生成予定パス: {0}".format(output_multi_split_vmd_path), decoration=MLogger.DECORATION_BOX)

    # フォーム無効化
    def disable(self):
        self.vmd_file_ctrl.disable()
        self.model_file_ctrl.disable()
        self.output_vmd_file_ctrl.disable()
        self.bone_target_btn_ctrl.Disable()
        self.multi_split_btn_ctrl.Disable()
        self.remove_unnecessary_flg_ctrl.Disable()

    # フォーム無効化
    def enable(self):
        self.vmd_file_ctrl.enable()
        self.model_file_ctrl.enable()
        self.output_vmd_file_ctrl.enable()
        self.bone_target_btn_ctrl.Enable()
        self.multi_split_btn_ctrl.Enable()
        self.remove_unnecessary_flg_ctrl.Enable()

    def on_doubleclick(self, event: wx.Event):
        self.timer.Stop()
        logger.warning("ダブルクリックされました。", decoration=MLogger.DECORATION_BOX)
        event.Skip(False)
        return False

    def on_convert_multi_split(self, event: wx.Event):
        self.timer.Start(200)
        self.Bind(wx.EVT_TIMER, self.on_convert, id=TIMER_ID)

    # 多段分割変換
    def on_convert(self, event: wx.Event):
        self.timer.Stop()
        self.Unbind(wx.EVT_TIMER, id=TIMER_ID)
        # フォーム無効化
        self.disable()
        # タブ固定
        self.fix_tab()
        # コンソールクリア
        self.console_ctrl.Clear()
        # 出力先を多段分割パネルのコンソールに変更
        sys.stdout = self.console_ctrl

        self.vmd_file_ctrl.save()
        self.model_file_ctrl.save()

        # JSON出力
        MFileUtils.save_history(self.frame.mydir_path, self.frame.file_hitories)

        self.elapsed_time = 0
        result = True
        result = self.vmd_file_ctrl.is_valid() and result
        result = self.model_file_ctrl.is_valid() and result

        if len(self.bone_target_txt_ctrl.GetValue()) == 0:
            logger.error("分割対象ボーンが指定されていません。", decoration=MLogger.DECORATION_BOX)
            result = False

        if not result:
            # 終了音
            self.frame.sound_finish()
            # タブ移動可
            self.release_tab()
            # フォーム有効化
            self.enable()

            return result

        # 多段分割変換開始
        if self.multi_split_btn_ctrl.GetLabel() == "多段分割停止" and self.convert_multi_split_worker:
            # フォーム無効化
            self.disable()
            # 停止状態でボタン押下時、停止
            self.convert_multi_split_worker.stop()

            # タブ移動可
            self.frame.release_tab()
            # フォーム有効化
            self.frame.enable()
            # ワーカー終了
            self.convert_multi_split_worker = None
            # プログレス非表示
            self.gauge_ctrl.SetValue(0)

            logger.warning("多段分割を中断します。", decoration=MLogger.DECORATION_BOX)
            self.multi_split_btn_ctrl.SetLabel("多段分割")

            event.Skip(False)
        elif not self.convert_multi_split_worker:
            # フォーム無効化
            self.disable()
            # タブ固定
            self.fix_tab()
            # コンソールクリア
            self.console_ctrl.Clear()
            # ラベル変更
            self.multi_split_btn_ctrl.SetLabel("多段分割停止")
            self.multi_split_btn_ctrl.Enable()

            self.convert_multi_split_worker = MultiSplitWorkerThread(self.frame, MultiSplitThreadEvent, self.frame.is_saving, self.frame.is_out_log)
            self.convert_multi_split_worker.start()

            event.Skip()
        else:
            logger.error("まだ処理が実行中です。終了してから再度実行してください。", decoration=MLogger.DECORATION_BOX)
            event.Skip(False)

        return result

    # 多段分割変換完了処理
    def on_convert_multi_split_result(self, event: wx.Event):
        self.elapsed_time = event.elapsed_time
        logger.info("\n処理時間: %s", self.show_worked_time())
        self.multi_split_btn_ctrl.SetLabel("多段分割")

        # 終了音
        self.frame.sound_finish()

        # タブ移動可
        self.release_tab()
        # フォーム有効化
        self.enable()
        # ワーカー終了
        self.convert_multi_split_worker = None
        # プログレス非表示
        self.gauge_ctrl.SetValue(0)
        # ラベル変更
        self.multi_split_btn_ctrl.SetLabel("多段分割")
        self.multi_split_btn_ctrl.Enable()

    def show_worked_time(self):
        # 経過秒数を時分秒に変換
        td_m, td_s = divmod(self.elapsed_time, 60)

        if td_m == 0:
            worked_time = "{0:02d}秒".format(int(td_s))
        else:
            worked_time = "{0:02d}分{1:02d}秒".format(int(td_m), int(td_s))

        return worked_time


#!/usr/bin/env python

import tkinter as tk
import tkinter.ttk as ttk
import zmq
import threading
import json
import logging

"""

Frame Layout
|----|---------|--------|
|Main|Markdown |Html    |
|Menu|Input    |Preview |
|    |         |        |
|    |         |        |
|    |         |        |
|----|---------|--------|

"""

class MainMenu(ttk.Frame):
	"""Contains buttons to read from and write to the database."""
	def __init__(self, parent, *args, **kwargs):
		super().__init__(parent, *args, **kwargs)
		self.parent = parent
		self.logger = logging.getLogger(__name__+'.'+self.__class__.__name__)

		self.connect_button = ttk.Button(self, text="Connect to DB", command=self.fill_post_list)
		self.connect_button.pack(side="top")

		self.posts_listbox = tk.Listbox(self,
				selectmode=tk.SINGLE,
				exportselection=True)
		self.posts_listbox.pack(side="top")
		self.posts_listbox.bind("<<ListboxSelect>>", self.select_post)

		self.new_post_button = ttk.Button(self, text="New Post", command=self.new_post)
		self.new_post_button.pack(side="top")

		self.published = tk.BooleanVar()
		self.published.set(False)
		self.publish_checkbutton = ttk.Checkbutton(self,
				text="Publish",
				var=self.published,
				command=self.publish_post)
		self.publish_checkbutton.pack(side="top")

	def select_post(self, event):
		"""Retrieve selected post from database and display content in
		markdown preview.
		"""
		try:
			post_id = self.post_ids[self.posts_listbox.curselection()[0]]
			self.logger.debug(f'Selected post {post_id}.')
		except:
			pass
		else:
			post = self.parent.request(
				'read_post',
				{'post_id': post_id})
			self.parent.post_id = post_id
			self.parent.display_md(post['post_content']['content'])
			self.parent.display_title(post['title'])

	def fill_post_list(self):
		"""Get all posts from the database and fill the listbox with
		datetime and title. Also map listbox indices to post_ids.
		"""
		self.posts_listbox.delete(0, tk.END)
		posts = self.parent.request('read_all_posts',{})
		self.post_ids = []
		[(self.post_ids.append(post['post_id']),
		  self.posts_listbox.insert(tk.END, f"{post['datetime_created']} - {post['title']}"))
			for post in posts]
		self.logger.debug('Posts list filled.')

	def new_post(self):
		"""Create new post in listbox."""
		post = self.parent.request('new_post', {})
		self.post_ids.append(post['post_id'])
		self.posts_listbox.insert(tk.END, "UNPUBLISHED - TITLE")
		self.posts_listbox.selection_set(tk.END)
		self.posts_listbox.see(tk.END)
		self.posts_listbox.event_generate("<<ListboxSelect>>")

	def update_title(self, title):
		"""Update title in listbox."""
		index = self.post_ids.index(self.parent.post_id)
		old_title = self.posts_listbox.get(index)
		new_title = old_title.replace("TITLE", title)
		self.posts_listbox.delete(index)
		self.posts_listbox.insert(index, new_title)
		self.posts_listbox.selection_set(index)
		self.posts_listbox.see(index)
		self.logger.debug(f'Update title: {new_title}')

	def publish_post(self):
		pass

class MarkdownInput(ttk.Frame):
	"""Text input area to display and write markdown for a blog post."""
	def __init__(self, parent, *args, **kwargs):
		super().__init__(parent, *args, **kwargs)
		self.parent = parent
		self.logger = logging.getLogger(__name__+'.'+self.__class__.__name__)

		self.title_entry = tk.Entry(self)
		self.title_entry.pack()
		self.title_entry.bind("<Return>", self.update_title)

		self.md_input = tk.Text(self,
				exportselection=False)
		self.md_input.pack()
		self.md_input.bind("<<Modified>>", self.update_md)

		self.old_content = ''

	def display_md(self, md_string):
		"""Populate markdown preview area with the given string."""
		if md_string is not None:
			self.md_input.delete('1.0', tk.END)
			self.md_input.insert('1.0', md_string)
			self.old_content = md_string
			self.md_input.edit_modified(False)
			self.logger.debug('Display content.')

	def update_md(self, event=None):
		"""Write updated markdown to database."""
		new_content = event.widget.get('1.0', tk.END).rstrip('\n\r')
		if new_content != self.old_content:
			self.old_content = new_content
			updated_post = self.parent.request(
				'update_content',
				{'post_id': self.parent.post_id, 'content': new_content, 'publish': False})
			self.md_input.edit_modified(False)
			self.logger.debug(f'Updated content: {new_content}')

	def update_title(self, event=None):
		"""Handle title updates."""
		title = event.widget.get()
		self.parent.update_title(title)
		self.logger.debug(f'Update title: {title}')

	def display_title(self, title):
		"""Display new title string."""
		if title is None:
			title = 'UNTITLED'
		self.title_entry.delete(0, tk.END)
		self.title_entry.insert(0, title)
		self.logger.debug(f'Display new title: {title}')

class HtmlPreview(ttk.Frame):
	"""Text area to display HTML code generated using markdown."""
	def __init__(self, parent, *args, **kwargs):
		super().__init__(parent, *args, **kwargs)
		self.parent = parent
		self.logger = logging.getLogger(__name__+'.'+self.__class__.__name__)

		self.html_preview = tk.Text(self)
		self.html_preview.pack()

	#TODO

class Main(ttk.Frame):
	"""Base frame to contain MainMenu, MarkdownInput and HtmlPreview frames."""
	def __init__(self, parent, socket, *args, **kwargs):
		super().__init__(parent, *args, **kwargs)
		self.parent = parent
		self.logger = logging.getLogger(__name__+'.'+self.__class__.__name__)

		self.main_menu = MainMenu(self)
		self.main_menu.pack(side="left")
		self.md_input = MarkdownInput(self)
		self.md_input.pack(side="right")
		self.html_preview = HtmlPreview(self)
		self.html_preview.pack(side="right")

		self.socket = socket

		# Current listbox selection
		self.post_id = 0
		# Current post title
		self.post_title = ''

	def display_md(self, md_string):
		"""Forward to MarkdownInput."""
		self.md_input.display_md(md_string)

	def display_title(self, title):
		"""Forward to MarkdownInput."""
		self.md_input.display_title(title)

	def update_title(self, title):
		"""Forward to MainMenu."""
		self.post_title = title
		self.main_menu.update_title(title)

	def request(self, header, args):
		"""Encapsulate the request - reply cycle.

		header - The request itself.
		args - Optional arguments to that request.
		"""
		self.socket.send_json(
			{'header': header,
			 'args': args})
		self.logger.debug(f'Request header: {header}, args: {args}.')
		reply = self.socket.recv_json()
		self.logger.debug(f'Reply: {str(reply)[:20]}...')
		return reply

class BloggerGUI():
	"""Handles launching and closing the Main frame."""
	def __init__(self, zmq_context=None):
		self.logger = logging.getLogger(__name__+'.'+self.__class__.__name__)
		if zmq_context is not None:
			self.launch(zmq_context)

	def launch(self, zmq_context):
		"""Establish connection to controller over zeroMQ socket and
		launch tk mainloop.
		"""
		self.logger.debug("Launching GUI...")
		self.socket = zmq_context.socket(zmq.REQ)
		self.socket.connect("inproc://req_handler")

		self.root = tk.Tk()
		Main(self.root, self.socket).pack(side="top", fill="both", expand=True)
		self.root.protocol("WM_DELETE_WINDOW", self.on_close)
		self.root.mainloop()

	def on_close(self):
		"""Send exit signal to controller and quit the mainloop."""
		self.socket.send_json(
			{'header': 'exit',
			 'args': {}})
		self.socket.recv()
		self.root.quit()
		self.logger.debug('GUI closed.')

def launch_gui(zmq_context):
	"""Launch GUI in its own thread."""
	gui = BloggerGUI()
	gui_thread = threading.Thread(target=gui.launch, args=(zmq_context,))
	gui_thread.start()

if __name__ == "__main__":
	launch_gui()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import paho.mqtt.client as mqtt_paho
#给paho.mqtt.client定义一个别名
from flask import Flask,request,send_from_directory,redirect          #从flask Web框架里导入flask,request...类
from werkzeug import secure_filename                                  #从werkzeug实用函数库导入secure_filname(上传文件)
import w
import laser

client = mqtt_paho.Client()
flaskapp = Flask(__name__)#app是Flask的实例，它接收包或者模块的名字作为参数，但一般都是传递__name__让函数通过传入这个名字确定程序的根目录，以便获得静态文件和模板文件的目录

@flaskapp.route('/static/<path:path>')                                 #/static/xxx的url处理（url是一个网页地址）
def send_static(path):
    return send_from_directory('static', path)                         #从static文件夹下返回结果文件

@flaskapp.route('/open/<string:door>')                          #/open/xxx的url处理
def api_open(door):
    client.publish('/control/Warehouse/open', door, qos=1)      #给仓库发布命令开门

@flaskapp.route('/upload', methods=['POST'])                   #POST方式处理上传文件
def upload_file():
    if 'file1' in request.files:                                   #上传图片
        request.files['file1'].save('file1.jpg')                   #保存到文件file1.jpg
    if 'file2' in request.files:                                   #上传数据
        request.files['file2'].save('file2.db')                   #保存到文件file2.db
        try:
            w.run()
            laser.run()
        except e:
            print(e)
    return"上传成功"

def on_message(conn, userdata, msg):                        #订阅消息处理
    print(msg.topic+" "+str(msg.payload))

client.on_message = on_message
client.connect("192.168.12.1", 1883, 60)                         #连接树莓派

client.subscribe("/values/Warehouse/#", 1)                     #仓库订阅多层消息
client.loop_start()
flaskapp.run(host="0.0.0.0")                            #执行app.run就可以启动服务了。默认Flask只监听虚拟机的本地127.0.0.1这个地址，端口为5000

'''
请在python代码旁边创建一个static文件夹
并将文件放到static文件夹下面，然后可以通过以下地址访问
http://127.0.0.1:5000/static/文件名

例如图片文件“081200.png”对应
http://127.0.0.1:5000/static/081200.png
网页文件“test.html”对应
http://127.0.0.1:5000/static/test.html

访问如下地址可以发送mqtt
http://127.0.0.1:5000/open/1
可以将链接指向该地址用于开门，例如：
这是图片

上传文件例子
http://127.0.0.1:5000/static/upload.html

下载文件例子
把minecraft.exe放到static下面
'''


import json
import sys, operator

class TruthDiscovery():

    def __init__(self, matrix):

        self.matrix = matrix
        self.num_sources = len(matrix)
        self.num_measured_variables = self.calcTotNumMeasuredVariables()

        self.a = self.initAi()
        self.b = self.initBi()

        self.d = 0.5

        self.Z = dict.fromkeys(range(1,self.num_measured_variables+2))
        self.H = dict.fromkeys(range(1,self.num_measured_variables+2))
        self.E = dict.fromkeys(self.matrix.keys())

    def numMeasuredVariables(self, source_id):
        return len(self.matrix[source_id])

    def calcTotNumMeasuredVariables(self):
        max_var = 0
        for source_id in self.matrix:
            sorted_measured_variables = sorted(self.matrix[source_id].keys())
            max_var = max(max_var, sorted_measured_variables[-1])
        return max_var-1

    def calcSi(self, i):
        return len(self.matrix[i]) / float(self.num_measured_variables)

    def initAi(self):
        a = {}
        for i in self.matrix.keys():
            a[i] = self.calcSi(i)
        return a

    def initBi(self):
        b = {}
        for i in self.matrix.keys():
            b[i] = self.calcSi(i) * 0.5
        return b

    def calcNextAi(self, s):
        numerator = sum([self.Z[j] for j in self.Z if j in self.matrix[s]])
        denominator = sum(self.Z.values())
        return numerator / float(denominator)

    def calcNextBi(self, s):
        Ki = self.numMeasuredVariables(s)
        N = self.num_measured_variables
        numerator = sum([self.Z[j] for j in self.Z if j in self.matrix[s]])
        denominator = sum(self.Z.values())
        return (Ki - numerator) / float(N - denominator)

    def calcNextDi(self, s):
        numerator = sum(self.Z.values())
        return numerator / float(self.num_measured_variables)

    def calcA(self, j):
        PI = 1
        for i in self.matrix:
            if j in self.matrix[i]:
                PI *= self.a[i]
            else:
                PI *= (1-self.a[i])
        return PI

    def calcB(self, j):
        PI = 1
        for i in self.matrix:
            if j in self.matrix[i]:
                PI *= self.b[i]
            else:
                PI *= (1-self.b[i])
        return PI

    def calcZ(self, j):
        numerator = self.calcA(j) * self.d
        denominator = self.calcA(j) * self.d + self.calcB(j) * (1 - self.d)
        return numerator / float(denominator)

    def calcSourceReliability(self, s):
        return self.a[s] * self.d / float(self.calcSi(s))

    def mle(self):
        t = 0
        while t < 20:

            for j in self.Z:
                self.Z[j] = self.calcZ(j)

            for i in self.matrix:
                self.a[i] = self.calcNextAi(i)
                self.b[i] = self.calcNextBi(i)
                self.d = self.calcNextDi(i)
            t += 1

        for j in self.Z:
            if self.Z[j] >= 0.5:
                self.H[j] = 1
            else:
                self.H[j] = 0

        for i in self.matrix:
            self.E[i] = self.calcSourceReliability(i)

    def verify(self, ground_file):
        truth_dict = {}
        f = open(ground_file)
        for line in f:
            measured_variable_id, correctness_indicator = line.split(',')
            truth_dict[int(measured_variable_id)] = int(correctness_indicator)
        f.close()
        incorr_count = 0
        corr_count = 0
        for j in truth_dict:
            if truth_dict[j] != self.H[j]:
                print('Incorrect:', j, truth_dict[j], self.H[j])
                incorr_count += 1
            else:
                corr_count +=1
        print('\n#Correct: {}, #Incorrect: {}\n'.format(corr_count, incorr_count))

    def dumpOptimalDecision(self, filename):
        f = open(filename, 'w')
        for j in self.H:
            f.write(str(j) + ',' + str(self.H[j]) + '\n')
        f.close()

    def writeProbs(self, filename):
        f = open(filename, 'w')
        for tup in sorted(self.Z.items(), key=operator.itemgetter(1), reverse=True):
            f.write(str(tup[0]) + ',' + str(tup[1]) + '\n')
        f.close()
        print('\nSuccessfully wrote probabilities to {}\n'.format(filename))


from django.urls import path, re_path, include

from securedcontent import views, api

api_urlpatterns = [
    path('stats', api.StatsView.as_view()),
    path('content', api.ContentView.as_view()),
    re_path('^open/(?P<pk>\w+)', api.ContentDetailView.as_view()),
]

urlpatterns = [
    path('api/', include(api_urlpatterns)),

    path('', views.CreateContentView.as_view(), name='create-content',),
    re_path(r'^open/(?P<pk>\w+)', views.OpenContentView.as_view(), name='open'),
]


import requests
import yaml
import uuid
import os,sys,stat
import logging
import logging.config
import time
import glob
import random

def readconfig(pathtoconfig):
    with open(pathtoconfig, 'r') as f:
        return yaml.load(f)

def save_a_file(directory,name,content):
    fullpath = os.path.join(directory,name)
    fo = open(fullpath, "wb")
    fo.write(content);
    fo.close()
    return fullpath

def find_output_to_forward(jobdirroot):
    dirs = glob.glob(os.path.join(jobdirroot,"*/F_*"))
    if dirs:
        return dirs[0]
    else:
        return False

def mark_job_as_forwarded(jobdir):
    wfdir = os.path.dirname(jobdir)
    jobdirname = os.path.basename(jobdir)
    newjobdir = "D_"+jobdirname[2:]
    os.rename(jobdir,os.path.join(wfdir,newjobdir))
    return newjobdir

def forward_outputs(jobdir):
    inputs = readconfig(os.path.join(jobdir,"inputs.yaml"))
    outputs = confapp["outputs"]

    forward_finished = True
    for ind, out in enumerate(outputs):

        genfilename = "genfiles-"+out["name"]+".yaml"
        if os.path.exists(os.path.join(jobdir,genfilename)):
            log.debug("Loading list of generated files for \""+out["name"]+"\"")
            genfiles = readconfig(os.path.join(jobdir,genfilename))
        else:
            log.debug("Creating list of generated files for \""+out["name"]+"\"")
            if "filter" in out:
                fullpath = os.path.join(jobdir,"sandbox",out["filter"])
            else:
                fullpath = os.path.join(jobdir,"sandbox",out["name"])
            log.debug("Filter is: \""+fullpath+"\"")
            generated_files = glob.glob(fullpath)
            generated_files.sort()
            genfiles = {}
            genfiles['count'] = len(generated_files)
            genfiles['files'] = [ {"index":outind,"name":os.path.basename(x)} for outind,x in enumerate(generated_files) ]
            save_a_file(jobdir,genfilename,yaml.dump(genfiles,default_flow_style=False))
        log.debug("Num of generated files for \""+out["name"]+"\" is "+str(genfiles['count']))

        target_forward_count_filename = os.path.join(jobdir,'target-forward-count-'+str(ind))
        if os.path.exists(target_forward_count_filename):
            target_forward = readconfig(target_forward_count_filename)
        else:
            target_forward = {}
            target_forward['portcount'] = 0
            target_forward['ips'] = []
            save_a_file(jobdir,target_forward_count_filename,yaml.dump(target_forward,default_flow_style=False))
        log.debug("Target forward count for link "+str(ind)+" is "+str(target_forward['portcount']))

        if os.path.exists(os.path.join(jobdir,'inputs.yaml')):
            inputsyaml = readconfig(os.path.join(jobdir,'inputs.yaml'))
            index_list = inputsyaml['indexes']['out_file_indxs']
            count_list = inputsyaml['indexes']['out_file_maxs']
        else:
            index_list = [0]
            count_list = [1]

        log.debug("Index list: "+str(index_list))
        log.debug("Count list: "+str(count_list))
        log.debug("Genfiles: "+str(genfiles))

        for one_genfile in genfiles['files']:
            if one_genfile['index'] < target_forward['portcount']:
                continue

            log.debug("Preparing sending "+str(one_genfile)+" to link "+str(ind))
            content = {}
            content['wfid'] = str(inputs['wfid'])
            content['inputs'] = []
            one_input = {}
            one_input['name'] = out["targetname"]

            if genfiles['count'] > 1:
                one_input['index_list'] = index_list+[one_genfile['index']]
                one_input['count_list'] = count_list+[genfiles['count']]
            else:
                one_input['index_list'] = index_list
                one_input['count_list'] = count_list

            itemcount=1
            for i in range(len(one_input['count_list'])):
                itemcount=itemcount*one_input['count_list'][i]

            multiplier = 1
            itemindex = 0
            for i in range(len(one_input['index_list'])-1,-1,-1):
                itemindex =  itemindex + one_input['index_list'][i] * multiplier
                multiplier = multiplier * one_input['count_list'][i]

            one_input['index'] = itemindex
            one_input['count'] = itemcount
            one_input['post_file'] = out['targetname']
            content['inputs'].append(one_input)
            r = out['targetip']
            if isinstance(r, list):
                targetiplist = [x.encode('ascii', 'ignore').split("'")[1] for x in r]
                if 'distribution' in out:
                    distr = out['distribution']
                    if 'random' == distr:
                        targetiplist = [targetiplist[random.randint(0, len(targetiplist)-1)]]
            else:
                targetiplist = [r]
            log.info('Will send file to ips: {}'.format(targetiplist))
            log.debug("Content is: \n"+str(content))

            for targetip in targetiplist:
                if targetip in target_forward['ips']:
                    continue
                try:
                    url = 'http://' + targetip + ':' + str(out['targetport']) + '/flowbster'
                    log.info("Sending content for \""+str(out.get("targetname"))+"\" to \""+url+"\"")
                    yaml_id = str(uuid.uuid4())
                    payload = {'yaml': yaml_id}
                    files = {out["targetname"]: open(os.path.join(jobdir,"sandbox",one_genfile['name']), 'rb'), yaml_id: yaml.dump(content)}
                    r = requests.post(url, params=payload, files=files)
                    target_forward['ips'].append(targetip)
                except:
                    log.exception('Exception occured during sending content:')
                    log.info("Sending: "+str(targetip)+" on port "+str(out['targetport'])+" is inaccessible. Will retry later.")
                    forward_finished = False
            if forward_finished:
                target_forward['portcount']+=1
                target_forward['ips']=[]
                save_a_file(jobdir,target_forward_count_filename,yaml.dump(target_forward,default_flow_style=False))
                log.info("Sending: SUCCESS.")
            else:
                save_a_file(jobdir,target_forward_count_filename,yaml.dump(target_forward,default_flow_style=False))
                log.info("Sending: FAILED. Will retry later.")
                break
    if forward_finished:
        mark_job_as_forwarded(jobdir)
        return True
    else:
        return False

def forward_one_output():
    log.info("Looking for an output to be forwarded at \""+jobdirroot+"\"")
    jobdir = find_output_to_forward(jobdirroot)
    if jobdir:
        log.info("Found output to forward at \""+jobdir+"\"")
        ret = forward_outputs(jobdir)
        log.info("Forward finished.")
        return ret
    else:
        log.info("No output found to be forwarded.")
        return False

def loadconfig(sysconfpath):
    global confsys, jobdirroot, log, confapp
    confsys = readconfig(sysconfpath)
    jobdirroot = os.path.join(confsys['jobdirroot'])
    if not os.path.exists(jobdirroot): os.makedirs(jobdirroot)
    logging.config.dictConfig(confsys['logging'])
    log = logging.getLogger("flowbster.forwarder")
    confapp = readconfig(confsys['appconfigpath'])

if len(sys.argv)==3 and sys.argv[1]=="-c":
    loadconfig(sys.argv[2])
else:
    loadconfig(os.path.join('/etc','flowbster-config-sys.yaml'))

while True:
    if forward_one_output()==False :
        time.sleep(confsys['sleepinterval'])


import tensorflow as tf
import keras
import math
from models import VGG19_2
from sklearn.metrics import accuracy_score
from tensorflow.keras.utils import to_categorical
import matplotlib.pyplot as plt
import numpy as np
from kerassurgeon.operations import delete_channels

NUM_CLASSES = 10
EPOCHS = 75
BATCH = 32
INPUT_SHAPE = (32, 32, 3)
NUMBER_OF_SAMPLES = 200   # out of 10000
TRESH_HOLD = 50

# RGB
cifar = tf.keras.datasets.cifar10
(x_train, y_train), (x_test, y_test) = cifar.load_data()

x_train, x_test = x_train / 255.0, x_test / 255.0  # To make the data between 0~1
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)

x_train, x_validation = x_train[0:40000], x_train[40000:50000]
y_train, y_validation = y_train[0:40000], y_train[40000:50000]
x_test_light = x_test[0:NUMBER_OF_SAMPLES]
print(x_train.shape, 'train samples')
print(y_train.shape, 'train labels')
print(x_validation.shape, 'validation samples')
print(y_validation.shape, 'validation labels')
print(x_test.shape, 'test samples')
print(y_test.shape, 'train labels')
print(x_train[0].shape)

opt = keras.optimizers.RMSprop(learning_rate=0.0001, decay=1e-6)

model_origin = VGG19_2(INPUT_SHAPE, NUM_CLASSES)
model_origin.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
model_origin.load_weights('weights_path.h5')
total_params = model_origin.count_params()
total_filters = 2*64 + 2*128 + 4*256 + 4*512 + 4*512

"""Pruning with sum sorting algorithm"""
after_prun_pararms_vec = []
precent_pruned_params_vec = []
precent_pruned_filters_vec = []
acc_score_vec = []

for precent in range(100, 55, -5):
    total_pruned_filters = 0
    model = model_origin
    block_precentage_pruning = [0, 0, 0, precent/100, precent/100]

    i = 0
    vec = []
    for layer in model.layers:
        name = str(layer)
        if 'Conv2D' in name:
            vec.append(i)
        i = i+1
    layer_number = iter(vec)

    #  The layer we prun in the weights matrix. - you need to check what layer to prun in weights = model3.get_weights()
    conv_layers = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
    Prun_1 = math.floor(block_precentage_pruning[0]*64)
    Prun_2 = math.floor(block_precentage_pruning[1]*128)
    Prun_3 = math.floor(block_precentage_pruning[2]*256)
    Prun_4 = math.floor(block_precentage_pruning[3]*512)
    Prun_5 = math.floor(block_precentage_pruning[4]*512)

    prun_vec = []
    for x in range(0, 2):
        prun_vec.append(Prun_1)
    for x in range(2, 4):
        prun_vec.append(Prun_2)
    for x in range(4, 8):
        prun_vec.append(Prun_3)
    for x in range(8, 12):
        prun_vec.append(Prun_4)
    for x in range(12, 16):
        prun_vec.append(Prun_5)

    prun_list = iter(prun_vec)

    weights = model.get_weights()

    idx = 0
    for x in conv_layers:
        curr_prun_number = next(prun_list)
        layer_number_iter = next(layer_number)
        if curr_prun_number == 0:
            idx = idx+1
            continue

        prediction = model.predict(x_test_light)
        layer_shape = weights[x].shape
        output_layer = []
        neuron_num = layer_shape[-1]
        identical_counter = np.zeros((neuron_num, neuron_num), dtype=int)

        for i in range(0, NUMBER_OF_SAMPLES):  # comparing all samples
            for j in range(0, neuron_num):
                for k in range(j+1, neuron_num):
                    if np.sum(abs(prediction[idx][i, :, :, j] - prediction[idx][i, :, :, k])) < TRESH_HOLD:
                        identical_counter[j][k] = identical_counter[j][k] + 1

        treshhold_pruning = (precent / 100) * NUMBER_OF_SAMPLES
        filters_to_prun = np.zeros((512 * 512, 2))
        PrunedNeurons = np.zeros((512 * 512, 1), dtype=int)
        idx_2 = 0
        for j in range(0, neuron_num):
            for k in range(j+1, neuron_num):
                if identical_counter[j][k] > treshhold_pruning:
                    filters_to_prun[idx_2] = (j, k)
                    PrunedNeurons[idx_2] = k
                    idx_2 = idx_2 + 1
        PrunedNeurons = np.unique(PrunedNeurons)
        PrunedNeurons = np.delete(PrunedNeurons, 0, 0)
        layer = model.layers[layer_number_iter]
        model = delete_channels(model, layer, PrunedNeurons)
        model_origin.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
        weights = model.get_weights()
        layer_shape_after = weights[x].shape
        neurons_after = layer_shape_after[-1]
        total_pruned_filters = total_pruned_filters + (neuron_num - neurons_after)
        idx = idx + 1
    after_prun_pararms = model.count_params()
    after_prun_pararms_vec.append(after_prun_pararms)
    precent_pruned_params = (total_params - after_prun_pararms) / total_params * 100
    precent_pruned_params_vec.append(precent_pruned_params)
    precent_pruned_filters = total_pruned_filters / total_filters * 100
    precent_pruned_filters_vec.append(precent_pruned_filters)
    scores = model.predict(x_test)
    y_pred = np.argmax(scores[18], axis=1)
    y_true = np.argmax(y_test, axis=1)
    acc_score = accuracy_score(y_true, y_pred)
    acc_score_vec.append(acc_score)
    print('Accuracy:')
    print(acc_score)
plt.plot(precent_pruned_params_vec, acc_score_vec)
plt.ylabel('Accuracy')
plt.xlabel('prunning of network params %')
plt.show()
plt.plot(precent_pruned_filters_vec, acc_score_vec)
plt.ylabel('Accuracy')
plt.xlabel('prunning of network filters %')
plt.show()

f = open('Results_path.txt', "a")
f.write("{} \n\n{} \n\n{} \n\n{} \n\n".format(after_prun_pararms_vec, precent_pruned_params_vec, precent_pruned_filters_vec, acc_score_vec))
f.close()


import subprocess as sp
import os
import shlex

class TestPrerequisites():

    def test_payu_exists(self):
        """
        Test that there is a payu installation.

        To get this test to pass.
        git clone https://github.com/marshallward/payu.git
        Then follow installation instructions.
        """

        cmd = 'which payu'
        sp.check_output(shlex.split(cmd))

    def test_experiments_exist(self):
        """
        Test that the payu experiment repo has been downloaded.

        To get this test to pass, run either:

        git clone https://github.com/cwsl/payu-experiments.git experiments

        or ./setup.py
        """

        assert(os.path.exists('./experiments/.git'))

    def test_ocean_ice_inputs(self):
        """
        Test that all the necessary ocean and ice inputs exist.

        To get this test to pass, either:

        Type payu init --laboratory ./lab --model access
        Then populate lab/input with the correct model inputs.
        You may find these at /short/public/access-om/

        or, ./setup.py
        """

        inputs = ['cice_om_360x300', 'cice_om_1440x1080',
                  'mom_om_360x300', 'mom_om_1440x1080',
                  'oasis_om_360x300', 'oasis_om_1440x1080',
                  'core2_nyf_matm']

        for i in inputs:
            assert(os.path.exists(os.path.join('lab/input', i)))


from sys import path

path.insert(0, '../')
from common.nlputils import NLPUtils
from common.senna.sennawrapper import SennaWrapper
from common.triple import Triple

class SemanticRoleLabeler:

    def extract(self, input_filename, output_filename, verbose=False):
        return self.__senna(input_filename, output_filename, verbose)

    def __senna(self, input_filename, output_filename, verbose=False):
        if verbose:
            print('Performing Sentence Role Labeling with SENNA...')

        senna = SennaWrapper()

        out_contents = ''
        with open(input_filename, 'r') as input_file:
            sentence_number = 0
            for line in input_file.readlines():
                if len(line) < 1:
                    continue

                senna_output = senna.srl(line, verbose=False)
                for predicate in senna_output.keys():
                    pred_args = senna_output[predicate]
                    pred_arg_names = NLPUtils.get_verbnet_args(predicate, verbose)
                    if len(pred_arg_names) < 1:
                        print('WARNING -- Unable to retrieve predicate arg names for "{}"'.format(predicate))

                    if verbose:
                        print('predicate: {}, args: {}'.format(predicate, pred_args))

                    for pred_arg in pred_args: # iterating over list of tuples
                        if pred_arg[0].startswith('AM-'): # Adjuncts
                            #if 'AM-NEG' == pred_arg[0]: # Check https://etd.ohiolink.edu/!etd.send_file?accession=osu1430876809&disposition=inline
                            #    predicate = 'not {}'.format(predicate)
                            #else:
                            #    predicate = ' '.join([pred_arg[1].strip(), predicate])

                            # Remove initial stopwords (e.g. determiners)
                            s = pred_arg[1].strip()
                            split = s.split(' ', 1)

                            while NLPUtils.is_stopword(split[0]) and len(split) > 1:
                                s = s.split(' ', 1)[1]
                                split = s.split(' ')

                            triple = Triple(sentence_number, predicate, 'local:{}'.format(pred_arg[0]), s)
                            if verbose:
                                print(triple.to_string())

                            out_contents += triple.to_string() + '\n'

                        elif pred_arg[0].startswith('C-'): # Continuities
                            print('Ignoring Continuity')
                        elif pred_arg[0].startswith('R-'): # References
                            print('Ignoring Reference')
                        elif pred_arg[0].startswith('V'): # Verbs
                            print('Ignoring Verb')
                        else: # Arguments (A0, A1, A2, etc.)
                            # Remove initial stopwords (e.g. determiners)
                            s = pred_arg[1].strip()
                            split = s.split(' ', 1)

                            while NLPUtils.is_stopword(split[0]) and len(split) > 1:
                                s = s.split(' ', 1)[1]
                                split = s.split(' ')

                            role_index = int(pred_arg[0][-1])
                            try:
                                role = pred_arg_names[role_index]
                            except IndexError:
                                print('No role index found for {}. Falling back to defaults!'.format(pred_arg[0]))
                                if role_index == 0:
                                    role = 'subject'
                                elif role_index == 1:
                                    role = 'object'
                                elif role_index == 2:
                                    role = 'indirect_object'
                                else:
                                    role = 'other'
                            triple = Triple(sentence_number, predicate, 'vn.role:{}'.format(role), s)
                            if verbose:
                                print(triple.to_string())

                            out_contents += triple.to_string() + '\n'

                sentence_number += 1

            input_file.close()

        with open(output_filename, 'w') as output_file:
            output_file.write(out_contents)
            output_file.close()

        return output_filename


from ase.io import read
from flosic_os import xyz_to_nuclei_fod,ase2pyscf
from pyscf import dft,gto
from flosic_scf import FLOSIC
import numpy as np

# This example visualizes how an electric field can be applied to DFT and FLO-SIC calculations. The application of the electric field is done to calculate the polarizability alpha.
# alpha will be calculated by approximating the derivative of the dipole moment mu with a finite difference like approach.
# The testing system will be an H4 structure (two H2 molecules.)
# First, we need the calculator objects and therefore mole objects.

molecule = read('H4.xyz')
geo,nuclei,fod1,fod2,included =  xyz_to_nuclei_fod(molecule)
spin = 0
charge = 0
b = 'cc-pVQZ'
xc = 'LDA,PW'
mol = gto.M(atom=ase2pyscf(nuclei), basis=b,spin=spin,charge=charge)

# Now we can initiliaze the calculator objects.

dft_object = dft.UKS(mol)
dft_object.xc = xc
sic_object = FLOSIC(mol,xc=xc,fod1=fod1,fod2=fod2,ham_sic='HOO')

# To calculate alpha we first calculate mu WITHOUT an electric field.

dft_object.kernel()
dft_mu0 = dft_object.dip_moment()[-1]
sic_object.kernel()
sic_mu0 = sic_object.dip_moment()[-1]

# Now we apply the electric field to the DFT and FLO-SIC calculator objects.
# For this we first set the gauge origin for the dipole integral.

mol.set_common_orig([0., 0., 0.])

# Then we calculate the new external potential.

vext = (mol.intor('cint1e_kin_sph') + mol.intor('cint1e_nuc_sph') + np.einsum('x,xij->ij', (0,0,0-0.001), mol.intor('cint1e_r_sph', comp=3)))

# Now we update this external potential for both calculators.

dft_object.get_hcore = lambda *args: vext
sic_object.get_hcore = lambda *args: vext

# Now we determine mu WITH electric field applied.

dft_object.kernel()
dft_mu1 = dft_object.dip_moment()[-1]
sic_object.kernel()
sic_mu1 = sic_object.dip_moment()[-1]

# With this we can calculate alpha.

alpha_dft = (dft_mu1 - dft_mu0)/(-0.001)
alpha_sic = (sic_mu1 - sic_mu0)/(-0.001)

# Now we can output the results.
# NOTE: We need to convert to atomic units.

print('Polarizability of H4 (DFT): %0.1f (should be %0.1f)'  %(alpha_dft*0.393456,37.2707068494))
print('Polarizability of H4 (FLO-SIC): %0.1f (should be %0.1f)' % (alpha_sic*0.393456,34.0331661325))
print('Polarizability of H4 (Reference Value): %0.1f' %(29.5))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct  9 03:13:49 2019

@author: altsai
"""

import os
import sys
import numpy as np
import csv
import pandas as pd
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import match_coordinates_sky
from astropy.table import Table
from astropy.wcs import WCS

dir_refstar='./RefStar/'

file_list_in=dir_refstar+'gasp_RefStar.list'
#print(file_list_in)
#df_list=pd.read_csv(file_list_in,sep='\t',skiprows=[0,1,2,17,18])
df_list=pd.read_csv(file_list_in,sep='\t',skiprows=[0,1,2,19,20])
print(df_list)
#sys.exit(0)

df_list_recommend=df_list.loc[df_list['NoteIdx']<2].reset_index(drop=True)
n_recommend=len(df_list_recommend)
#sys.exit(0)

regname=['']*n_recommend
obj_ra_hhmmss=['']*n_recommend
obj_dec_ddmmss=['']*n_recommend
star_ra_hhmmss=['']*n_recommend
star_dec_ddmmss=['']*n_recommend
star_ra_deg=[0]*n_recommend
star_dec_deg=[0]*n_recommend
#star_ra_hhmmssdius=['']*n_recommend
#sys.exit(0)

for i in range(n_recommend):
    obj_name=df_list_recommend['ObjectName'][i]
#    regfile='RefStar_'+obj_name+'_fk5.reg'
    regfile='RefStar_'+obj_name+'_bkg_fk5.reg'
    regname[i]=regfile
#    print(regname[i])
    obj_ra_hhmmss[i]=df_list_recommend['RA_hhmmss'][i].replace(' ',':')
    obj_dec_ddmmss[i]=df_list_recommend['DEC_ddmmss'][i].replace(' ',':')
#    print(obj_ra_hhmmss[i],obj_dec_ddmmss[i])
    id_star=df_list_recommend['RefStarID'][i]
#    print(id_star)
    cmd_search_refstar='cat '+dir_refstar+regfile+' |grep "text={'+id_star+'}"| cut -d "(" -f2 | cut -d ")" -f1'
    print(cmd_search_refstar)
    radecr=os.popen(cmd_search_refstar,"r").read().splitlines()[0].split(',')
#    print(radecr)
    ra_hhmmss=radecr[0]
    dec_ddmmss=radecr[1]
#    radius=radecr[2].split('"')[0]
#    print(ra,dec,radius)
    star_ra_hhmmss[i]=ra_hhmmss
    star_dec_ddmmss[i]=dec_ddmmss
#    star_ra_hhmmssdius[i]=radius
#    print(ra_hhmmss,dec_ddmmss)
    star_skycoord=SkyCoord(ra_hhmmss,dec_ddmmss,frame='icrs',unit=(u.hourangle,u.deg))
#    print(star_skycoord)
    star_ra_deg[i]=star_skycoord.ra.deg
    star_dec_deg[i]=star_skycoord.dec.deg
#    print(star_skycoord.ra.deg)
#    print(star_skycoord.dec.deg)

#sys.exit(0)
df_list_recommend['RA_hhmmss']=obj_ra_hhmmss
df_list_recommend['DEC_ddmmss']=obj_dec_ddmmss

df_list_recommend.insert(5,'RefStarregname',regname)
df_list_recommend.insert(7,'RefStarRA_hhmmss',star_ra_hhmmss)
df_list_recommend.insert(8,'RefStarDEC_ddmmss',star_dec_ddmmss)
#df_list_recommend.insert(9,'RefStarRadius_as',star_ra_hhmmssdius)
df_list_recommend.insert(9,'RefStarRA_deg',star_ra_deg)
df_list_recommend.insert(10,'RefStarDEC_deg',star_dec_deg)

print(df_list_recommend)

file_list_out='gasp_refStar_radec.txt'
df_list_recommend.to_csv(file_list_out,sep='|')


s="Hi He Lied Because Boron Could Not Oxidize Fluorine. New Nations Might Also Sign Peace Security Clause. Arthur King Can."

s=s.split(' ')
a={}
for i in range(len(s)):
    if i==1 or i==5 or i==6 or i==7 or i==8 or i==9 or i==15 or i==16 or i==19:
        a[i]=s[i][0:2]
    else:
        a[i]=s[i][0]

for i in range(len(a)):
    print(a[i])


class Solution:
    def reorganizeString(self, S: str) -> str:
        letters = defaultdict(int)
        res = [None] * len(S)
        for s in S:
            letters[s] += 1

        last = ''
        letterMax = ['', 0]
        for p in range(len(S)):
            letterMax[1] = 0
            for l in letters:
                if letters[l] > letterMax[1] and last != l:
                    letterMax = [l, letters[l]]
            if last == letterMax[0]:
                return ""
            last = letterMax[0]
            res[p] = letterMax[0]
            letters[letterMax[0]] -= 1
        return ''.join(res)


import re

from pathlib import Path
from collections import defaultdict

import click

import numpy as np

from imageio import imread, imsave

from skimage.morphology import label
from skimage.measure import regionprops

def imsave_f(fn, image):

    imsave(fn, 255 * image.astype(np.uint8))

def grouper(n, iterable):
    args = [iter(iterable)] * n
    return map(list, zip(*args))

def spike(fpath):

	im = imread(fpath).astype(np.float32)

	result = (im[:,:,0] > 240) & (im[:,:,1] > 210)  & (im[:,:,2] > 140)

	imsave('result.png', 255 * result.astype(np.uint8))

def sorted_nicely(l):
    """Return list sorted in the way that humans expect.

    :param l: iterable to be sorted
    :returns: sorted list
    """
    convert = lambda text: int(text) if text.isdigit() else text
    sort_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
    return sorted(l, key=sort_key)

def extract_seed_series(fpath):

    # c, r = 2194, 254
    # c, r = 550, 1442
    c, r = 1594, 1382
    ts = 64

    fns = sorted_nicely([str(fn) for fn in fpath.iterdir()])

    sections = []
    for i in range(40, 240, 10):
        im = imread(fns[i])
        section = im[r-ts:r+ts,c-ts:c+ts]
        seed = (section.astype(np.float32)[:,:,0] - section.astype(np.float32)[:,:,2]) > 20
        radicle = (section[:,:,0] > 240) & (section[:,:,1] > 210)  & (section[:,:,2] > 140)
        section[np.where(~seed)] = [0, 0, 0]
        result = np.zeros(section.shape)
        result[np.where(radicle)] = [0, 255, 0]
        sections.append(np.vstack((section, result)))

    # n_rows = 4
    # # joined = np.block(list(grouper(n_rows, sections)))
    joined = np.hstack(sections)

    imsave('joined.png', joined)

def get_seed_centroids(mask_fpath):

    # Load and convert to grayscale
    im = imread(mask_fpath).max(axis=2)
    ccs = label(im)

    return [tuple(map(int, r.centroid)) for r in regionprops(ccs, coordinates='rc')]

def draw_red_border(im):
    xdim, ydim, _ = im.shape

    im[0:xdim,0]      = 255, 0, 0
    im[0:xdim,ydim-1] = 255, 0, 0
    im[0,0:ydim]      = 255, 0, 0
    im[xdim-1,0:ydim] = 255, 0, 0

def process_from_mask(mask_fpath, data_fpath):

    centroids = get_seed_centroids(mask_fpath)

    ts = 64

    fns = sorted_nicely([str(fn) for fn in data_fpath.iterdir()])

    b = 20
    cols = []
    for i in range(100, 250, 10):
        im = imread(fns[i])
        sections = []
        for r, c in centroids:
            section = np.copy(im[r-ts:r+ts,c-ts:c+ts])
            sub = section[b:2*ts-b,b:2*ts-b,:]
            radicle = (sub[:,:,0] > 240) & (sub[:,:,1] > 210)  & (sub[:,:,2] > 140)
            if np.sum(radicle) > 2:
                draw_red_border(section)
            sections.append(section)
        cols.append(np.vstack(sections))

    result = np.hstack(cols)

    imsave('result.png', result)

def score_each_seed(mask_fpath, data_fpath):

    centroids = get_seed_centroids(mask_fpath)

    ts = 64

    fns = sorted_nicely([str(fn) for fn in data_fpath.iterdir()])

    first_appearance = defaultdict(int)
    b = 20

    for i in range(100, 250, 1):
        im = imread(fns[i])
        for r, c in centroids:
            section = im[r-ts:r+ts,c-ts:c+ts]
            sub = section[b:2*ts-b,b:2*ts-b,:]
            radicle = (sub[:,:,0] > 240) & (sub[:,:,1] > 210)  & (sub[:,:,2] > 140)
            if np.sum(radicle) > 2:
                centroids.remove((r, c))
                first_appearance[(r, c)] = i

    boxid=1
    m = min(first_appearance.values())
    print('{},{}'.format('box', 'time'))
    for t in first_appearance.values():
        print('{},{}'.format(boxid, t-m))

@click.command()
@click.argument('mask_fpath')
@click.argument('data_fpath')
def main(mask_fpath, data_fpath):

    process_from_mask(
        Path(mask_fpath),
        Path(data_fpath)
    )
	# extract_seed_series(Path(fpath))

if __name__ == '__main__':
	main()


#================================================
# Steamworks For Python
#================================================
from ctypes import *
import sys, os

# User status
#------------------------------------------------
FriendFlags = {  # regular friend
    'None': 0x00,
    'Blocked': 0x01,
    'FriendshipRequested': 0x02,
    'Immediate': 0x04,
    'ClanMember': 0x08,
    'OnGameServer': 0x10,
    'RequestingFriendship': 0x80,
    'RequestingInfo': 0x100,
    'Ignored': 0x200,
    'IgnoredFriend': 0x400,
    'Suggested': 0x800,
    'All': 0xFFFF,
    }

# Main Steam Class, obviously
#------------------------------------------------
class Steam:
	# Set some basic variables for the Steam class
	cdll = None
	warn = False
	loaded = False

	# Initialize Steam
	@staticmethod
	def Init():
		os.environ['LD_LIBRARY_PATH'] = os.getcwd()
		# Check system architecture
		if sys.maxsize > 2**32 is False:
			OS_BIT = '32bits'
		else:
			OS_BIT = '64bits'
		# Loading SteamPy API for Linux
		if sys.platform == 'linux' or sys.platform == 'linux2':
			Steam.cdll = CDLL(os.path.join(os.getcwd(), "libsteampy.so"))
			print("SteamPy loaded for Linux")
			Steam.loaded = True
		# Loading SteamPy API for Mac
		elif sys.platform == 'darwin':
			Steam.cdll = CDLL(os.path.join(os.getcwd(), "libsteampy.dylib" ))
			print("SteamPy loaded for Mac")
			Steam.loaded = True
		# Loading Steamworks API for Windows
		elif sys.platform == 'win32':
			# Check Windows architecture
			if OS_BIT == '32bits':
				Steam.cdll = CDLL(os.path.join(os.getcwd(), "SteamworksPy.dll"))
			else:
				Steam.cdll = CDLL(os.path.join(os.getcwd(), "SteamworksPy64.dll"))
			print("Steamworks loaded for Windows")
			Steam.loaded = True
		# Unrecognized platform, warn user and do not load Steam API
		else:
			print("SteamPy failed to load (unsupported platform!")
			Steam.warn = True
			return

		# Set restype for initialization
		Steam.cdll.IsSteamRunning.restype = c_bool
		# Check that Steam is running
		if Steam.cdll.IsSteamRunning():
			print("Steam is running!")
		else:
			print("Steam is not running!")

		# Boot up the Steam API
		Steam.cdll.SteamInit()

		# Set restype for Friends functions
		Steam.cdll.GetFriendCount.restype = int
		Steam.cdll.GetPersonaName.restype = c_char_p
		# Set restype for User functions
		Steam.cdll.GetSteamID.restype = int
		# Set restype for User Statistic functions
		Steam.cdll.GetAchievement.restype = bool
		Steam.cdll.GetStatInt.restype = int
		Steam.cdll.GetStatFloat.restype = c_float
		Steam.cdll.ResetAllStats.restype = bool
		Steam.cdll.RequestCurrentStats.restype = bool
		Steam.cdll.SetAchievement.restype = bool
		Steam.cdll.SetStatInt.restype = bool
		Steam.cdll.SetStatFloat.restype = bool
		Steam.cdll.StoreStats.restype = bool

	@staticmethod
	def Call(method):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return method()

# Class for Steam Friends
#------------------------------------------------
class SteamFriends:

	@staticmethod
	def GetFriendCount(flag=FriendFlags['All']):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.GetFriendCount(flag)

	@staticmethod
	def GetPlayerName():
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.GetPersonaName()

# Class for Steam Users
#------------------------------------------------
class SteamUser:

	@staticmethod
	def GetPlayerID():
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.GetSteamID()

# Class for Steam User Statistics
#------------------------------------------------
class SteamUserStats:

	@staticmethod
	def GetAchievement(name):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.GetAchievement(name)

	@staticmethod
	def GetStatFloat(name):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.GetStatFloat(name)

	@staticmethod
	def GetStatInt(name):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.GetStatInt(name)

	@staticmethod
	def ResetAllStats(achievesToo):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.ResetAllStats(achievesToo)

	@staticmethod
	def RequestCurrentStats():
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.RequestCurrentStats()

	@staticmethod
	def SetAchievement(name):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.SetAchievement(name)

	@staticmethod
	def SetStat(name, value):
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			if isinstance(value, float):
				return Steam.cdll.SetStatFloat(name, value)
			elif isinstance(value, int):
				return Steam.cdll.SetStatInt(name, value)
			else:
				raise Exception("SteamUserStats: SetStat value can be only int or float.")

	@staticmethod
	def StoreStats():
		if not Steam.cdll and not Steam.warn:
			print("Steam is not loaded")
			Steam.warn = True
			return False
		else:
			return Steam.cdll.StoreStats()


# -*- coding: utf-8 -*-

"""
a class with graph utils
level: method (correspond to a .dot file)
dot version: use abstracted jimple representation; all call relations (in project + jdk + 3rd party) inside the dot file
graph version: use int number (0, 1, 2 ...) as node id, different node id may contains same label.
"""

import logging
import random
import networkx as nx
import matplotlib.pyplot as plt
from build_data.preprocess import preprocessor

logger = logging.getLogger('cg2vec')

__author__ = 'zfy'

LOGFORMAT = "%(asctime).19s %(levelname)s %(filename)s: %(lineno)s %(message)s"

class Graph:

    """
    basic implementation & extension of networkx.MultiDiGraph()
    """

    def __init__(self, graph):
        """
        receive a networkx.graph after relabelling from dot files
        use int number as node id
        :param graph: a relabelled networkx.graph()
        """
        self.graph = graph

    def nodes(self):
        """return all node id of the graph"""
        return list(self.graph.nodes())

    def edges(self):
        """return all edges of a graph"""
        return list(self.graph.edges())

    def get_label(self, node):
        """return the label of a node"""
        return self.graph.nodes[node]['label']

    def get_nl(self, node):
        """return natural language of a node"""
        return self.graph.nodes[node]['natural_language']

    def get_lineNum(self,node):
        """return line number of a node in the graph"""
        if self.graph.nodes[node].get('lineNum') != None:
            return self.graph.nodes[node]['lineNum']
        else:
            return -1

    def get_name(self, node):
        """return name of a node"""
        return self.graph.nodes[node]['name']

    def adjacency_iter(self):
        """
        get the adjacent list of a multi-directed-graph
        note that if there are multi lines between two nodes, only one is recorded in a adj_list
        e.g. (0, 1, color='red) and (0, 1, color='blue') --> 0: 1
        TODO: will multi-lines affect random walk?
        :return: dict -> {node : its adj_list}
        """
        adj_dict = {}
        for node, neibour in self.graph.adjacency():
            adj_dict.setdefault(node, [])
            for v in neibour:
                adj_dict[node].append(v)
        return adj_dict

    def subgraph(self):
        #TODO
        return None

    def make_undirected(self):
        #TODO
        return None

    def make_consistent(self):
        #TODO
        return None

    def remove_self_loop(self):
        """
        remove self-loop in graph
        e.g. (1 -> 1)
        :return refined adjacent dict
        """

        removed_num = 0

        adjacent = self.adjacency_iter()
        for node, neibour in adjacent:
            if node in neibour:
                neibour.remove(node)
                removed_num += 1

        logger.info('remove self-loops: removed {} self loops'.format(removed_num))
        return adjacent

    def check_self_loop(self):
        """
        check if a graph has any self-loop
        :return: true or false
        """

        adjacent = self.adjacency_iter()
        for node, neibour in adjacent:
            if node in neibour:
                return True

        return False

    def has_edge(self):
        #TODO
        return None

    def degree(self):
        #TODO
        return None

    def order(self):
        #TODO
        return None

    def number_of_edges(self):
        #TODO
        return None

    def number_of_nodes(self):
        #TODO
        return None

    def out_edges(self, node):
        """return all out edges of a node"""
        return list(self.graph.out_edges(node))

    def in_edges(self, node):
        """return all out edges of a node"""
        return list(self.graph.in_edges(node))

    def out_edges_multi(self, node):
        """return all out edges in 3-tuple form"""
        out_edges_3_form = []
        out_edges = self.out_edges(node)
        for edge in out_edges:
            s = edge[0]
            t = edge[1]
            multi = self.graph[s][t]
            for i in range(len(multi)):
                label = self.graph[s][t][i]['label']
                out_edges_3_form.append((s,t,label))
        return out_edges_3_form

    def in_edges_multi(self, node):
        """return all in edges in 3-tuple form"""
        in_edges_3_form = []
        in_edges = self.in_edges(node)
        for edge in in_edges:
            s = edge[0]
            t = edge[1]
            multi = self.graph[s][t]
            for i in range(len(multi)):
                # print('i: ' + str(i))
                # print(s, t)
                label = self.graph[s][t][i]['label']
                in_edges_3_form.append((s, t, label))
        return in_edges_3_form

    def find_tail_node(self, head, edge):
        """
        given a head node and a edge, find the tail node
        :param head: head node
        :param edge: edge of 3-tuple form: (u, v, k='id')
        :return: tail node
        """
        s = edge[0]
        t = edge[1]
        return t

    def draw(self):
        plt.subplot(221)
        nx.draw(self.graph, with_labels=True, font_weight='bold')
        plt.show()

    @staticmethod
    def draw_save(graph, path):
        plt.subplot(211)
        nx.draw(graph, with_labels=True, font_weight='bold')
        # plt.show()
        plt.savefig(path)

    def random_walk_adjacent(self, path_length, alpha=0, rand=random.Random(), start=None):
        """
        returns a truncated random walk.
        :param path_length: length of a random walk
        :param alpha: probability of restart
        :param rand: random seed
        :param start: the start node of random walk
        :return: a generated random walk
        """
        if start:
            path = [start]
        else:
            # sampling nodes not edges
            path = [rand.choice(self.nodes())]

        while (len(path) < path_length):
            cur = path[-1]
            cur_neibour = self.adjacency_iter().get(cur)
            if len(cur_neibour) > 0:
                if rand.random() > alpha:
                    path.append(rand.choice(cur_neibour))
                else:
                    path.append(path[0])
            else:
                # this is how we truncate the walk
                break

        return [self.get_label(node) for node in path]

    def random_walk_graph(self, path_length, alpha=0, rand=random.Random(), start=None):
        """
        returns a truncated random walk that use original graph
        :param path_length: length of a random walk
        :param alpha: probability of restart
        :param rand: random seed
        :param start: the start node of random walk
        :return: a generated random walk
        """

        if start:
            path = [start]
        else:
            # sample a start node
            # sampling edges not nodes
            path = [rand.choice(self.nodes())]

        while (len(path) < path_length):
            cur = path[-1]
            out_edges = self.out_edges_multi(cur)
            if len(out_edges) > 0:
                if rand.random() > alpha:
                    selected_edge = rand.choice(out_edges)
                    next_node = self.find_tail_node(cur, selected_edge)
                    path.append(next_node)
                else:
                    path.append(path[0])
            else:
                break
        return [self.get_label(node) for node in path] # node, graph, dot_name

    def random_walk_graph_edge_sampled(self, path_length, alpha=0, rand=random.Random(), start=None):
        """
        returns a truncated random walk that use original graph, edges are sampled in walks
        :param path_length: length of a random walk
        :param alpha: probability of restart
        :param rand: random seed
        :param start: the start node of random walk
        :return: a generated random walk with edge sampled
        """

        if start:
            path = [start]
        else:
            # sample a start node
            # sampling edges not nodes
            path = [rand.choice(self.nodes())]

        edge_path = []
        while (len(path) < path_length):
            cur = path[-1]
            out_edges = self.out_edges_multi(cur)
            if len(out_edges) > 0:
                if rand.random() > alpha:
                    selected_edge = rand.choice(out_edges)
                    edge_path.append(selected_edge)
                    next_node = self.find_tail_node(cur, selected_edge)
                    path.append(next_node)
                else:
                    path.append(path[0])
            else:
                break
        res = []
        for i in range(0, len(path)-1):
            # print(self.get_label(path[i]))
            # print(edge_path[i][2])
            res.append(self.get_label(path[i]))
            res.append(edge_path[i][2])
        res.append(self.get_label(path[len(path)-1]))
        return res

    def reduce_graph_by_line_num(self):
        """
        1. given a line number, find all corresponding nodes : node_set
        2. find all edges link with node in node_set : edges_set = in_set + out_set
        3. remove inner edges : inner_set
        4. add a new node : new_node = label1 + label2 + ...
        5. attach remaining edges with new_node : remain_set = edges_set - inner_set
        6. delete inner nodes : inner_set
        7. relabel
        :return: relabelled graph
        """

        map = {}
        reversed_map = {}
        for node in self.graph.nodes():
            name = node
            line_num = self.get_lineNum(node)

            map.setdefault(name, [])
            map[name].append(line_num)
            reversed_map.setdefault(line_num, [])
            reversed_map[line_num].append(name)
        assert len(map.keys()) == len(self.graph.nodes())
        print(reversed_map)

        for num in reversed_map.keys():
            if int(num) != -1:
                # step1 : node_set
                node_set = reversed_map.get(num)

                # step2 : edges_set
                in_set = []
                out_set = []
                for node in node_set:
                    for e in self.out_edges_multi(node):
                        out_set.append(e)
                    for e in self.in_edges_multi(node):
                        in_set.append(e)
                edges_set = in_set + out_set
                edges_set = sorted(set(edges_set), key=edges_set.index)

                # step3 : edges_set = edges_set - inner_set
                inner_edges = []
                inner_nodes = reversed_map.get(num)
                # print(num +': ' + ' '.join(inner_nodes))
                for e in edges_set:
                    if e[0] in inner_nodes and e[1] in inner_nodes:
                        edges_set.remove(e)
                        inner_edges.append(e)

                # step4 : add a new graph
                new_node = inner_nodes[0] + '_'
                new_label = ''
                for node in inner_nodes:
                    l = self.get_label(node)
                    l = preprocessor.clean_label(l)
                    new_label += l + '\t'
                new_label = new_label.strip('\t')
                new_linenum = num
                self.graph.add_node(new_node, label=new_label, lineNum=new_linenum)

                # step5 : add new edges
                for e in edges_set:
                    self.graph.remove_edge(e[0], e[1])
                    if e in in_set:
                        source = e[0]
                        target = new_node
                        label = e[2]
                        self.graph.add_edge(source, target, label=label)

                    if e in out_set:
                        source = new_node
                        target = e[1]
                        label = e[2]
                        self.graph.add_edge(source, target, label=label)

                # step6 : remove inner nodes and edges
                self.graph.remove_nodes_from(inner_nodes)
                # for e in inner_edges:
                #     self.graph.remove_edge(e[0], e[1], e[2])

        return self.graph

        # '''relabel the graph'''
        # node_count_map = {}
        # count_label_map = {}
        # count = 0
        # for node in self.graph.nodes():
        #     label = self.get_label(node)
        #     node_count_map.setdefault(node, '')
        #     node_count_map[node] = str(count)
        #     count_label_map.setdefault(str(count), '')
        #     count_label_map[str(count)] = label
        #     count +=1
        # relabelled_graph = nx.relabel_nodes(self.graph, node_count_map)
        # for node in relabelled_graph.nodes():
        #     label = count_label_map.get(node)
        #     relabelled_graph.nodes[node]['label'] = label
        #
        # relabelled_graph = Graph(relabelled_graph)
        # # for node in relabelled_graph.nodes():
        # #     print(node, relabelled_graph.get_label(node))
        #
        # return relabelled_graph


# -*- coding: utf-8 -*-
#
# This file is part of REANA.
# Copyright (C) 2017, 2018 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""REANA Workflow Engine CWL reana pipeline."""

from __future__ import absolute_import, print_function, unicode_literals

import logging
import os
import pipes
import re
import shutil
import tempfile
import time
from pprint import pformat

import shellescape
from cwltool.command_line_tool import CommandLineTool
from cwltool.errors import WorkflowException
from cwltool.job import (JobBase, needs_shell_quoting_re,
                         relink_initialworkdir, stage_files)
from cwltool.pathmapper import ensure_writable
from cwltool.workflow import default_make_tool
from reana_commons.api_client import JobControllerAPIClient as rjc_api_client
from reana_commons.config import REANA_WORKFLOW_UMASK

from reana_workflow_engine_cwl.config import LOGGING_MODULE, MOUNT_CVMFS
from reana_workflow_engine_cwl.pipeline import Pipeline
from reana_workflow_engine_cwl.poll import PollThread

log = logging.getLogger(LOGGING_MODULE)

class ReanaPipeline(Pipeline):
    """REANA Pipeline class."""

    def __init__(self, **kwargs):
        """Constructor."""
        super(ReanaPipeline, self).__init__()
        self.service = rjc_api_client('reana-job-controller')
        if kwargs.get("basedir") is not None:
            self.basedir = kwargs.get("basedir")
        else:
            self.basedir = os.getcwd()

    def make_exec_tool(self, spec, loadingContext):
        """Make execution tool."""
        return ReanaPipelineTool(spec, loadingContext)

    def make_tool(self, spec, loadingContext):
        """Make tool."""
        if "class" in spec and spec["class"] == "CommandLineTool":
            return self.make_exec_tool(spec, loadingContext)
        else:
            return default_make_tool(spec, loadingContext)

class ReanaPipelineTool(CommandLineTool):
    """REANA Pipeline Tool class."""

    def __init__(self, spec, loadingContext):
        """REANA tool adapted to support REANA jobs submission."""
        super(ReanaPipelineTool, self).__init__(spec, loadingContext)

    def make_job_runner(self, runtimeContext):
        """REANA make job runner."""
        dockerReq, _ = self.get_requirement("DockerRequirement")
        if not dockerReq and runtimeContext.use_container:
            if runtimeContext.find_default_container:
                default_container = \
                    runtimeContext.find_default_container(self)
                if default_container:
                    self.requirements.insert(0, {
                        "class": "DockerRequirement",
                        "dockerPull": default_container
                    })

        return ReanaPipelineJob

class ReanaPipelineJob(JobBase):
    """REANA Pipeline Job."""

    def __init__(self, builder, joborder, make_path_mapper, requirements,
                 hints, jobname):
        """Constructor."""
        super(ReanaPipelineJob, self).__init__(
            builder, joborder, make_path_mapper, requirements, hints, jobname)

        self.builder = builder
        self.joborder = joborder
        self.make_path_mapper = make_path_mapper
        self.requirements = requirements
        self.hints = hints
        self.jobname = jobname

        self.outputs = None
        self.inplace_update = False
        self.volumes = []
        self.task_name_map = {}

    def add_volumes(self, pathmapper):
        """Customize add volumes for REANA system."""
        host_outdir = self.outdir
        container_outdir = self.builder.outdir
        for src, vol in pathmapper.items():
            if not vol.staged:
                continue
            if vol.target.startswith(container_outdir + "/"):
                host_outdir_tgt = os.path.join(
                    host_outdir, vol.target[len(container_outdir) + 1:])
            else:
                host_outdir_tgt = None
            if vol.type in ("File", "Directory"):
                if not vol.resolved.startswith("_:"):
                    resolved = vol.resolved
                    if not os.path.exists(resolved):
                        resolved = "/".join(
                            vol.resolved.split("/")[:-1]) + "/" + \
                            vol.target.split("/")[-1]
                    self.volumes.append((resolved, vol.target))
            elif vol.type == "WritableFile":
                if self.inplace_update:
                    self.volumes.append((vol.resolved, vol.target))
                else:
                    shutil.copy(vol.resolved, host_outdir_tgt)
                    ensure_writable(host_outdir_tgt)
            if vol.type == "WritableDirectory":
                if vol.resolved.startswith("_:"):
                    if not os.path.exists(vol.target):
                        os.makedirs(vol.target)
                else:
                    if self.inplace_update:
                        pass
                    else:
                        shutil.copytree(vol.resolved, host_outdir_tgt)
                        ensure_writable(host_outdir_tgt)
            elif vol.type == "CreateFile":
                if host_outdir_tgt:
                    with open(host_outdir_tgt, "wb") as f:
                        f.write(vol.resolved.encode("utf-8"))
                else:
                    fd, createtmp = tempfile.mkstemp(dir=self.tmpdir)
                    with os.fdopen(fd, "wb") as f:
                        f.write(vol.resolved.encode("utf-8"))

    def create_task_msg(self, working_dir, workflow_uuid):
        """Create job message spec to be sent to REANA-Job-Controller."""
        job_name = self.name
        docker_req, _ = self.get_requirement("DockerRequirement")
        container = str(docker_req['dockerPull'])
        umask_cmd = "umask {};".format(REANA_WORKFLOW_UMASK)
        requirements_command_line = umask_cmd
        for var in self.environment:
            requirements_command_line += "export {0}=\"{1}\";".format(
                var, self.environment[var])

        if self.volumes:
            for filepair in self.volumes:
                if os.path.isdir(filepair[0]):
                    requirements_command_line += "cp -a {0} {1} ;".format(
                        filepair[0], "/".join(filepair[1].split("/")[:-1]))
                else:
                    requirements_command_line += "cp -a {0} {1} ;".format(
                        filepair[0], filepair[1])

        mounted_outdir = self.outdir
        scr, _ = self.get_requirement("ShellCommandRequirement")

        shebang_lines = {"/bin/bash", "/bin/sh"}
        if scr or self.command_line[0] in shebang_lines:
            def shouldquote(x):
                return False
            self.command_line = self.command_line[2:]
        else:
            shouldquote = needs_shell_quoting_re.search
        shellQuote = True
        for b in self.builder.bindings:
            if "shellQuote" in b:
                shellQuote = b.get("shellQuote")
                break

        command_line = " ".join([shellescape.quote(arg) if shouldquote(arg)
                                 else arg for arg in
                                 self.command_line])
        command_line = command_line.replace('/bin/sh -c ', '')
        command_line = re.sub(
            "/var/lib/cwl/.*?/", "/".join(
                working_dir.split("/")[:-1]) +
            "/workspace/", command_line)
        command_line = re.sub(
            "/tmp/.*?/.*?/", working_dir + "/", command_line)
        if self.stdin:
            path = self.stdin.split("/")
            if os.path.isabs(self.stdin):
                command_line = command_line + " < {0}".format(self.stdin)
            else:
                if len(path) > 1:
                    parent_dir = "/".join(mounted_outdir.split("/")[:-1])
                    command_line = command_line + \
                        " < {0}".format(os.path.join(parent_dir, path[-1]))
                else:
                    command_line = command_line + \
                        " < {0}".format(os.path.join(mounted_outdir, path))
        if self.stdout:
            if os.path.isabs(self.stdout):
                command_line = command_line + " > {0}".format(self.stdout)
            else:
                command_line = command_line + \
                    " > {0}".format(os.path.join(
                        self.environment["HOME"], self.stdout))
        if self.stderr:
            if os.path.isabs(self.stderr):
                stderr = self.stderr
            else:
                stderr = os.path.join(mounted_outdir, self.stderr)
            command_line += " 2> " + stderr
            if scr and not shellQuote:
                command_line = command_line.replace("&2", stderr)

        wf_space_cmd = "mkdir -p {0} && cd {0} && ".format(
            self.environment["HOME"]) + command_line
        wf_space_cmd = requirements_command_line + wf_space_cmd

        docker_output_dir = None
        docker_req, _ = self.get_requirement("DockerRequirement")
        if docker_req:
            docker_output_dir = docker_req.get("dockerOutputDirectory", None)
        if docker_output_dir:
            wf_space_cmd = "mkdir -p {0} && {1} ; cp -r {0} {2}".format(
                docker_output_dir, wf_space_cmd, mounted_outdir)
        wf_space_cmd += "; cp -r {0}/* {1}".format(
            self.environment['HOME'], mounted_outdir)
        wrapped_cmd = "/bin/sh -c {} ".format(pipes.quote(wf_space_cmd))
        experiment = os.getenv("REANA_WORKFLOW_ENGINE_EXPERIMENT", "default")
        create_body = {
            "experiment": experiment,
            "image": container,
            "cmd": wrapped_cmd,
            "prettified_cmd": wrapped_cmd,
            "workflow_workspace": working_dir,
            "job_name": job_name,
            "cvmfs_mounts": MOUNT_CVMFS,
            "workflow_uuid": workflow_uuid
        }

        return create_body

    def run(self, runtimeContext):
        """Run a job."""
        self._setup(runtimeContext)

        env = self.environment
        if not os.path.exists(self.tmpdir):
            os.makedirs(self.tmpdir)
        vars_to_preserve = runtimeContext.preserve_environment
        if runtimeContext.preserve_entire_environment:
            vars_to_preserve = os.environ
        if vars_to_preserve is not None:
            for key, value in os.environ.items():
                if key in vars_to_preserve and key not in env:
                    env[key] = value
        env["HOME"] = self.builder.outdir
        env["TMPDIR"] = self.tmpdir
        if "PATH" not in env:
            env["PATH"] = os.environ["PATH"]
        if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ:
            env["SYSTEMROOT"] = os.environ["SYSTEMROOT"]

        try:
            stage_files(self.pathmapper, ignore_writable=True, symlink=False)
            if getattr(self, "generatemapper", ""):
                stage_files(self.generatemapper,
                            ignore_writable=self.inplace_update, symlink=False)
                relink_initialworkdir(self.generatemapper,
                                      self.outdir,
                                      self.builder.outdir,
                                      inplace_update=self.inplace_update)
        except OSError:
            # cwltool/process.py, line 239, in stage_files
            # shutil.copytree(p.resolved, p.target)
            pass
        self.add_volumes(self.pathmapper)
        if getattr(self, "generatemapper", ""):
            self.add_volumes(self.generatemapper)

        # useful for debugging
        log.debug(
            "[job %s] self.__dict__ in run() ----------------------" %
            (self.name)
        )
        log.debug(pformat(self.__dict__))

        task = self.create_task_msg(runtimeContext.working_dir,
                                    runtimeContext.workflow_uuid)

        log.info(
            "[job %s] CREATED TASK MSG----------------------" %
            (self.name)
        )
        log.info(pformat(task))

        try:
            # task_id = job_id received from job-controller
            task_id = runtimeContext.pipeline.service.submit(**task)
            task_id = str(task_id['job_id'])
            running_jobs = {"total": 1, "job_ids": [task_id]}
            runtimeContext.publisher.publish_workflow_status(
                runtimeContext.workflow_uuid, 1,
                message={
                    "progress": {
                        "running":
                        running_jobs,
                    }})
            log.info(
                "[job %s] SUBMITTED TASK ----------------------" %
                (self.name)
            )
            log.info("[job %s] task id: %s " % (self.name, task_id))
            self.task_name_map[self.name] = task_id
            operation = runtimeContext.pipeline.service.check_status(task_id)
        except Exception as e:
            log.error(
                "[job %s] Failed to submit task to job controller:\n%s" %
                (self.name, e)
            )
            return WorkflowException(e)

        def callback():
            try:
                outputs = self.collect_outputs(self.outdir)
                cleaned_outputs = {}
                for k, v in outputs.items():
                    if isinstance(k, bytes):
                        k = k.decode("utf8")
                    if isinstance(v, bytes):
                        v = v.decode("utf8")
                    cleaned_outputs[k] = v
                self.outputs = cleaned_outputs
                self.output_callback(self.outputs, "success")
            except WorkflowException as e:
                log.error("[job %s] job error:\n%s" % (self.name, e))
                self.output_callback({}, "permanentFail")
            except Exception as e:
                log.error("[job %s] job error:\n%s" % (self.name, e))
                self.output_callback({}, "permanentFail")
            finally:
                if self.outputs is not None:
                    log.info(
                        "[job %s] OUTPUTS ------------------" %
                        (self.name)
                    )
                    log.info(pformat(self.outputs))
                self.cleanup(runtimeContext.rm_tmpdir)

        poll = ReanaPipelinePoll(
            workflow_uuid=runtimeContext.workflow_uuid,
            task_id=self.task_name_map.get(self.name),
            jobname=self.name,
            service=runtimeContext.pipeline.service,
            operation=operation,
            callback=callback,
            publisher=runtimeContext.publisher
        )

        runtimeContext.pipeline.add_thread(poll)
        poll.start()

    def cleanup(self, rm_tmpdir):
        """Clean up procedure."""
        log.debug(
            "[job %s] STARTING CLEAN UP ------------------" %
            (self.name)
        )
        if self.stagedir and os.path.exists(self.stagedir):
            log.debug(
                "[job %s] Removing input staging directory %s" %
                (self.name, self.stagedir)
            )
            shutil.rmtree(self.stagedir, True)

        if rm_tmpdir:
            log.debug(
                "[job %s] Removing temporary directory %s" %
                (self.name, self.tmpdir)
            )
            shutil.rmtree(self.tmpdir, True)

class ReanaPipelinePoll(PollThread):
    """REANA Pipeline Poll class."""

    def __init__(self, workflow_uuid, task_id, jobname,
                 service, operation, callback, publisher):
        """Constructor."""
        super(ReanaPipelinePoll, self).__init__(operation)
        self.workflow_uuid = workflow_uuid
        self.task_id = task_id
        self.name = jobname
        self.service = service
        self.callback = callback
        self.publisher = publisher

    def run(self):
        """Start polling."""
        while not self.is_done(self.operation):
            time.sleep(self.poll_interval)
            # slow down polling over time till it hits a max
            # if self.poll_interval < 30:
            #     self.poll_interval += 1
            log.debug(
                "[job %s] POLLING %s" %
                (self.name, pformat(self.id))
            )
            try:
                self.operation = self.poll()
            except Exception as e:
                log.error("[job %s] POLLING ERROR %s" % (self.name, e))
                if self.poll_retries > 0:
                    self.poll_retries -= 1
                    continue
                else:
                    log.error("[job %s] MAX POLLING RETRIES EXCEEDED" %
                              (self.name))
                    break

        self.complete(self.operation)

    def poll(self):
        """Poll procedure."""
        return self.service.check_status(self.id)

    def is_done(self, operation):
        """Check if operation is done."""
        terminal_states = ["succeeded", "failed"]
        if operation['status'] in terminal_states:
            log.info(
                "[job %s] FINAL JOB STATE: %s ------------------" %
                (self.name, operation['status'])
            )
            if operation['status'] != "failed":
                # here we could publish that the job with id: self.task_id
                # succeeded or failed.
                self.publisher.publish_workflow_status(
                    self.workflow_uuid, 1, logs='',
                    message={
                        'progress': {
                            'finished':
                            {'total': 1, 'job_ids': [
                                self.task_id]},
                        }})
                log.error(
                    "[job %s] task id: %s" % (self.name, self.id)
                )
                log.error(
                    "[job %s] logs: %s" %
                    (
                        self.name,
                        self.service.get_logs(self.id)
                    )

                )
            else:
                self.publisher.publish_workflow_status(
                    self.workflow_uuid, 1, logs='',
                    message={
                        'progress': {
                            'failed':
                            {'total': 1, 'job_ids': [
                                self.task_id]},
                        }})
            return True
        return False

    def complete(self, operation):
        """Complete."""
        self.callback()


# Same comments given in tereceiver file
import pika
import sys

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='topic_logs', exchange_type='topic')

result = channel.queue_declare('', exclusive=True)
queue_name = result.method.queue

binding_key ='*.*.slow'
channel.queue_bind(exchange='topic_logs', queue=queue_name, routing_key=binding_key)

print(' [*] Waiting for logs '+ binding_key+'. To exit press CTRL+C')

def callback(ch, method, properties, body):
    print(" [x] %r:%r" % (method.routing_key, body))

channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True)

channel.start_consuming()

# Working @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@222