from libs.alibaba.alibaba import Alibaba

from libs.json import JSON

from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import WebDriverException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import StaleElementReferenceException
from selenium.webdriver.support.select import Select
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
import html
import time
import re
import os
import traceback
import types
import pendulum

class IneffectiveProduct:

    api = 'https://hz-mydata.alibaba.com/self/product/ineffective.htm'
    browser = None

    def __init__(self, market, account=None, headless=True, browser=None):
        self.market = market
        self.account = account
        self.lid = account['lid'] if account else market['lid']
        self.lpwd = account['lpwd'] if account else market['lpwd']
        self.lname = account['lname'] if account else market['lname']
        self.mobile = account['mobile'] if account else market['mobile']

        self.account = {}
        self.account['lid'] = self.lid
        self.account['lpwd'] = self.lpwd
        self.account['lname'] = self.lname
        self.account['mobile'] = self.mobile

        self.confi_dir = market['directory'] + '_config'
        self.visitor_dir = self.confi_dir + '//'+'visitors'

        self.alibaba = None
        self.browser = browser
        self.headless = headless

    def load_url(self):
        while True:
            try:
                if self.alibaba is None:
                    alibaba = Alibaba(self.lid, self.lpwd, market=self.market, headless=self.headless, browser=self.browser)
                    alibaba.login()
                    if self.browser is None:
                        self.browser = alibaba.browser
                    self.alibaba = alibaba

                self.browser.get(self.api)
                if 'login.alibaba.com' in self.browser.current_url:
                    self.alibaba.login()
                    self.browser.get(self.api)

                WebDriverWait(self.browser, 15).until(
                    EC.invisibility_of_element_located((By.CSS_SELECTOR, 'div.bp-loading-panel')))

                # try to close all follow-me-popups
                while True:
                    btn_close = self.browser.find_elements_by_css_selector('div.follow-me-close')
                    if btn_close:
                        webdriver.ActionChains(self.browser).send_keys(Keys.ESCAPE).perform()
                        # wait 1 second to see if new popup commes
                        self.browser.implicitly_wait(1)
                        continue
                    else:
                        break
                break
            except WebDriverException as e:
                if 'chrome not reachable' in str(e):
                    print(str(e))
                    self.browser = None
                continue

    def set_conditions(self, ineffective_time, owner):

        # set ineffective_time
        itime_dropdown_trigger = self.browser.find_element_by_css_selector('#J-ineffective-time div.ui-dropdown-trigger')
        itime_dropdown_trigger.click()
        itime_input = self.browser.find_element_by_css_selector('#J-ineffective-time-options input')
        ActionChains(self.browser).double_click(itime_input).perform()
        itime_input.send_keys(ineffective_time)
        self.browser.find_element_by_css_selector('#J-custom-time-OK').click()

        # set owner
        owner_dropdown_trigger = self.browser.find_element_by_css_selector('#J-product-owner .ui-dropdown-trigger')
        owner_dropdown_trigger.click()
        owner_lis = self.browser.find_elements_by_css_selector('ul[data-role="content"] li[data-role="item"]')
        for li in owner_lis:
            if owner.lower() in li.text.lower():
                li.click()
                break

        self.browser.find_element_by_css_selector('#J-preference-OK').click()
        self.wait_for_loading()

    def wait_for_loading(self):
        # WebDriverWait(self.browser, 15).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "#J-tip-loading")))

        try:
            WebDriverWait(self.browser, 1).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "#J-tip-loading")))
        except TimeoutException:
            pass
        WebDriverWait(self.browser, 15).until(EC.invisibility_of_element_located((By.CSS_SELECTOR, "#J-tip-loading")))

    def next_page(self):
        next_btn = self.browser.find_elements_by_css_selector('#J-pagination a.ui-pagination-next')
        if next_btn:
            next_btn[0].click()
            self.wait_for_loading()
            return True
        else:
            return False

    def crawl_current_page(self):
        products = []
        product_trs = self.browser.find_elements_by_css_selector('#J-product-statistics-content tr.product-item')
        for tr in product_trs:
            product = {}
            href = tr.find_element_by_css_selector('td.products-img a').get_attribute('href')
            product['ali_id'] = re.search('_(\d+)\.html',href).group(1)
            product['owner'] = tr.find_element_by_css_selector('td.products-owner').text.strip()
            product['ineffective_time'] = tr.find_element_by_css_selector('td.products-time').text.strip()
            product['update_time'] = tr.find_element_by_css_selector('td.products-updated').text.strip()
            products.append(product)
        return products

    def crawl(self, ineffective_time, owner, limit=None):
        products = {}

        self.load_url()
        self.set_conditions(ineffective_time, owner)

        while True:
            for product in self.crawl_current_page():
                products[product['ali_id']] = product
            if not self.next_page():
                break

            if limit and str(limit) == self.browser.find_element_by_css_selector('span.ui-pagination-active').text.strip():
                break
        return products


import random
import string
import datetime

'''
随机整数

fromInt <= res <= to
'''
def getRandomBetweenNumbers(fromInt , to) :
    return random.randint(int(fromInt), int(to));

'''
获取随机字符串
'''
def getRandomStr(length) :
    if (length <= 0):
        return None;
    limitLen = 60;
    realLen = length;
    if length > limitLen :
        perLen = 60;
        lastLen = 0;
        times = 1;
        # 不能整除
        times =int(length / limitLen) ;
        if (length % limitLen) > 0:
            lastLen = length - times * limitLen;

        ran_str = '';
        times = times + 1;
        for i in range(0 , times) :
            new_ran_str = '';
            if i == (times - 1) and lastLen > 0 :
                new_ran_str = ''.join(random.sample(string.ascii_letters + string.digits, lastLen))
            else:
                new_ran_str = ''.join(random.sample(string.ascii_letters + string.digits, perLen))
            ran_str = ran_str + new_ran_str;
        return ran_str;
    else:
        ran_str = ''.join(random.sample(string.ascii_letters + string.digits, length))
        return ran_str;

'''
返回字符串
'''
def getRandomBetweenNumbersStr(fromInt , to) :
    return str(random.randint(int(fromInt), int(to)));

'''
随机浮点数
'''
def getRandomFloatBetweenNumbers(fromInt , to) :
    return random.uniform(fromInt, to);

'''
获取随机数组
'''
def getRamdonByArr(arr):
    if arr and len(arr) > 0:
        idx = random.randint(0, len(arr) - 1);
        return arr[idx];
    return None;

'''
获取随机时间
'''
def getLimitRandomTime():
    startYear = '2017';
    endYear = '2017';

    startMonth = '5';
    endMonth = '7';

    startDate = '1';
    endDate = '28';

    time = '';
    time += getRandomBetweenNumbersStr(startYear, endYear);
    time += '-';

    month = getRandomBetweenNumbersStr(startMonth, endMonth);
    if int(month) < 10 :
        time += '0';
    time += month;
    time += '-';

    date = getRandomBetweenNumbersStr(startDate, endDate);
    if int(date) < 10 :
        time += '0';
    time += date;
    time += ' ';

    hour = getRandomBetweenNumbersStr(0, 23);
    if (int(hour) < 10) :
        time += '0';

    time += hour;
    time += ':';

    minute = getRandomBetweenNumbersStr(0, 59);
    if (int(minute) < 10) :
        time += '0';

    time += minute;
    time += ':';

    second = getRandomBetweenNumbersStr(0, 59);
    if (int(second) < 10) :
        time += '0';
    time += second;
    return time;

def getRandomTime():
    startYear = 2015;
    endYear = 2017;

    startMonth = 1;
    endMonth = 12;

    startDate = 1;
    endDate = 28;

    time = '';

    time += getRandomBetweenNumbersStr(startYear, endYear);
    time += '-';

    month = getRandomBetweenNumbersStr(startMonth, endMonth);
    if int(month) < 10 :
        time += '0';
    time += month;
    time += '-';

    date = getRandomBetweenNumbersStr(startDate, endDate);
    if int(date) < 10 :
        time += '0';
        time += date;
    time += ' ';

    hour = getRandomBetweenNumbersStr(0, 23);
    if (int(hour) < 10) :
        time += '0';

    time += hour;
    time += ':';

    minute = getRandomBetweenNumbersStr(0, 59);
    if (int(minute) < 10) :
        time += '0';

    time += minute;
    time += ':';

    second = getRandomBetweenNumbersStr(0, 59);
    if (int(second) < 10) :
        time += '0';
    time += second;
    return time;

def getRandomTodayTime():
    today = datetime.date.today();

    today = str(today) + ' ';
    random_hour = getRandomBetweenNumbers(0, 23);
    if random_hour < 10:
        today = str(today) + '0' + str(random_hour);
    else:
        today = str(today) + str(random_hour);
    today = today + ':';
    random_minite = getRandomBetweenNumbers(0, 59);
    if random_minite < 10:
        today = str(today) + '0' + str(random_minite);
    else:
        today = str(today) + str(random_minite);
    today = today + ':';
    random_second = getRandomBetweenNumbers(0, 59);
    if random_second < 10:
        today = str(today) + '0' + str(random_second);
    else:
        today = str(today) + str(random_second);

    return today;


import torch
from torch.utils.tensorboard import SummaryWriter

from args import *
from model_head import *
from dataloader_ytb import *
from loss_metric import *

writer = SummaryWriter()

model = ModelDisigner()
model = model.to(device)
model.load_state_dict(torch.load('pathignore/weights/test.pth'))

optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

print('Tensorboard graph name: ')
GRAPH_NAME = input()

iter = 0
for epoch in range(15):
	print('*'*10, 'epoch: ', epoch, '*'*10)
	for i, data in enumerate(train_loader):
		target, searchs, labels, depths, score_labels = data
		target, searchs, labels, depths, score_labels \
		 = target.to(device), searchs.to(device), labels.to(device), depths.to(device), score_labels.to(device)
		# try:
		pred_scores, pred_masks = model(target, searchs)
		loss = all_losses(pred_masks, labels, depths, pred_scores, score_labels)
		loss.backward()
		optimizer.step()
		optimizer.zero_grad()
		if iter % 10 == 0:
			print('iter: ', iter, 'loss: ', loss.mean().item())
			writer.add_scalars('%s_loss' % GRAPH_NAME, {'train' : loss.mean().item()}, iter)
		iter += 1
		if iter % 250 == 0:
			torch.save(model.state_dict(), 'pathignore/weights/test.pth')
			print('WEIGHTS IS SAVED: pathignore/weights/test.pth')

		# except RuntimeError:
		# 	pass
print('Tensorboard graph name: ', GRAPH_NAME)
writer.close()

torch.save(model.state_dict(), 'pathignore/weights/test.pth')
print('WEIGHTS IS SAVED: pathignore/weights/test.pth')


from plot_curves_window import curves as curves_window
from plot_curves_window_trend import curves as curves_window_trend
from shared_utils import make_county_dict

import os
import pandas as pd
import argparse
import shutil
import numpy as np

parser = argparse.ArgumentParser(description='NYI')

parser.add_argument(
    "--njobs",
    nargs=1,
    type=int,
    dest="njobs")

parser.add_argument(
    "--jobid",
    nargs=1,
    type=int,
    dest="jobid")

args = parser.parse_args()

# start = int(os.environ["SGE_DATE_ID"])
start = 1
county_dict = make_county_dict()
start_day = pd.Timestamp('2020-01-28') + pd.Timedelta(days=start)
year = str(start_day)[:4]
month = str(start_day)[5:7]
day = str(start_day)[8:10]

figures_path = "/p/project/covid19dynstat/autostart/BSTIM-Covid19_Window_Final/figures/{}_{}_{}/".format(year, month, day)
shared_path = "/p/project/covid19dynstat/shared_assets/figures/{}_{}_{}/".format(year, month, day)

counties = np.array_split(list(county_dict.keys()), args.njobs[0])[args.jobid[0]]
for c in counties:
    curves_window(start, c, n_weeks=3, model_i=35, save_plot=True)
    curves_window_trend(start, c, save_plot=True)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# File : core.classifiers.RCNLPTextClassifier.py
# Description : Echo State Network for text classification.
# Auteur : Nils Schaetti <nils.schaetti@unine.ch>
# Date : 01.02.2017 17:59:05
# Lieu : Nyon, Suisse
#
# This file is part of the Reservoir Computing NLP Project.
# The Reservoir Computing Memory Project is a set of free software:
# you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Foobar is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
#

import io
import argparse
from core.converters.RCNLPPosConverter import RCNLPPosConverter

if __name__ == "__main__":

    # Argument parser
    parser = argparse.ArgumentParser(description="RCNLP - Authorship attribution with Part-Of-Speech to Echo State Network")

    # Argument
    parser.add_argument("--file", type=str, help="Input text file")
    parser.add_argument("--lang", type=str, help="Language (ar, en, es, pt)", default='en')
    args = parser.parse_args()

    converter = RCNLPPosConverter()
    doc_array = converter(io.open(args.file, 'r').read())
    print(doc_array)
    print(doc_array.shape)
    print(doc_array[0])
    RCNLPPosConverter.display_representations(doc_array)
    data_set = RCNLPPosConverter.generate_data_set_inputs(doc_array, 2, 0)
    print(data_set)

# end if


"""pythonProject URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin
import helloworld

urlpatterns = [
    #django study
    url(r'^admin/', admin.site.urls),
    url(r'^polls/', include('polls.urls')),
    #url(r'^$', helloworld.RenderShortcut),
    url(r'^blog/', include('blog.urls')),
    url(r'', include('baseball.urls')),

    #django test
    url(r'^hello/$', helloworld.hello),
    url(r'^Smoothie/1/BackEndMessagingTest/SendMessage/$', helloworld.helloworldTest),
    url(r'^Smoothie/1/DjangoTest/$', helloworld.globalVarTest),
    url(r'^Add/$', helloworld.Add),
    url(r'^Sum/$', helloworld.Sum),
    url(r'^StaticTester/$', helloworld.staticTester),

]


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# build.py — build a sketch using arduino-builder
#
# Wrapper script around arduino-builder which accepts some ESP8266-specific
# options and translates them into FQBN
#
# Copyright © 2016 Ivan Grokhotkov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#

from __future__ import print_function
import sys
import os
import argparse
import subprocess
import tempfile
import shutil

def compile(tmp_dir, sketch, tools_dir, hardware_dir, ide_path, f, args):
    cmd = ide_path + '/arduino-builder '
    cmd += '-compile -logger=human '
    cmd += '-build-path "' + tmp_dir + '" '
    cmd += '-tools "' +  ide_path + '/tools-builder" '
    if args.library_path:
        for lib_dir in args.library_path:
            cmd += '-libraries "' + lib_dir + '" '
    cmd += '-hardware "' + ide_path + '/hardware" '
    if args.hardware_dir:
        for hw_dir in args.hardware_dir:
            cmd += '-hardware "' + hw_dir + '" '
    else:
        cmd += '-hardware "' + hardware_dir + '" '
    # Debug=Serial,DebugLevel=Core____
    cmd += '-fqbn=espressif:esp32:{board_name}:' \
            'FlashFreq={flash_freq},' \
            'UploadSpeed=921600'.format(**vars(args))
    cmd += ' '
    cmd += '-ide-version=10607 '
    cmd += '-warnings={warnings} '.format(**vars(args))
    if args.verbose:
        cmd += '-verbose '
    cmd += sketch

    if args.verbose:
        print('Building: ' + cmd, file=f)

    cmds = cmd.split(' ')
    p = subprocess.Popen(cmds, stdout=f, stderr=subprocess.STDOUT)
    p.wait()
    return p.returncode

def parse_args():
    parser = argparse.ArgumentParser(description='Sketch build helper')
    parser.add_argument('-v', '--verbose', help='Enable verbose output',
                        action='store_true')
    parser.add_argument('-i', '--ide_path', help='Arduino IDE path')
    parser.add_argument('-p', '--build_path', help='Build directory')
    parser.add_argument('-l', '--library_path', help='Additional library path',
                        action='append')
    parser.add_argument('-d', '--hardware_dir', help='Additional hardware path',
                        action='append')
    parser.add_argument('-b', '--board_name', help='Board name', default='esp32')
    parser.add_argument('-w', '--warnings', help='Compilation warnings level',
                        default='none', choices=['none', 'all', 'more'])
    parser.add_argument('-o', '--output_binary', help='File name for output binary')
    parser.add_argument('-k', '--keep', action='store_true',
                        help='Don\'t delete temporary build directory')
    parser.add_argument('--flash_freq', help='Flash frequency', default=40,
                        type=int, choices=[40, 80])
    parser.add_argument('sketch_path', help='Sketch file path')
    return parser.parse_args()

def main():
    args = parse_args()

    ide_path = args.ide_path
    if not ide_path:
        ide_path = os.environ.get('ARDUINO_IDE_PATH')
        if not ide_path:
            print("Please specify Arduino IDE path via --ide_path option"
                  "or ARDUINO_IDE_PATH environment variable.", file=sys.stderr)
            return 2

    sketch_path = args.sketch_path
    tmp_dir = args.build_path
    created_tmp_dir = False
    if not tmp_dir:
        tmp_dir = tempfile.mkdtemp()
        created_tmp_dir = True

    tools_dir = os.path.dirname(os.path.realpath(__file__)) + '/../tools'
    # this is not the correct hardware folder to add.
    hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores'

    output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin'
    if args.verbose:
        print("Sketch: ", sketch_path)
        print("Build dir: ", tmp_dir)
        print("Output: ", output_name)

    if args.verbose:
        f = sys.stdout
    else:
        f = open(tmp_dir + '/build.log', 'w')

    res = compile(tmp_dir, sketch_path, tools_dir, hardware_dir, ide_path, f, args)
    if res != 0:
        return res

    if args.output_binary is not None:
        shutil.copy(output_name, args.output_binary)

    if created_tmp_dir and not args.keep:
        shutil.rmtree(tmp_dir, ignore_errors=True)

if __name__ == '__main__':
    sys.exit(main())


#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

SITEURL = 'https://mattman-pelican-test.netlify.com'
SITELOGO = '/images/python-oregon-pdx-600.jpg'
AUTHOR = 'Pythonistas'
SITENAME = 'Python PDX West'
SITETITLE = 'Python PDX West'
SITESUBTITLE = 'Something clever here'

PATH = 'content'

TIMEZONE = 'America/Los_Angeles'

DEFAULT_LANG = 'en'

THEME = 'themes/Flex'

DISPLAY_PAGES_ON_MENU = False
LINKS_IN_NEW_TAB = False

#THEME = 'pelican-themes/clean-blog'
#THEME = 'pelican-themes/pelican-bootstrap3'
#BOOTSTRAP_THEME = 'flatly'
#DISPLAY_PAGES_ON_MENU = True

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

LINKS = (
    ('Meetup', 'https://www.meetup.com/Python-PDX-West/'),
    ('About', '/pages/about.html'),
    ('Twitter', 'https://twitter.com/PythonPdx'),
    ('Github', 'https://github.com/PythonPDX'),
)

TWITTER_USERNAME = "PythonPdx"
GITHUB_USERNAME = "PythonPDX"

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True


#!/usr/bin/env python
#-*- coding:utf-8 -*-

"""Overview:
  Classify cell candidates and determine true cells

Usage:
  HDoG_classifier.py PARAM_FILE

Options:
  -h --help        Show this screen.
  --version        Show version.
"""

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import pandas as pd
import scipy.ndimage
import os.path, joblib
import tifffile
import json
from docopt import docopt

from sklearn.mixture import BayesianGaussianMixture
from sklearn.tree import DecisionTreeClassifier
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.tree import export_graphviz
from sklearn.pipeline import Pipeline
import graphviz

from MergeBrain import WholeBrainCells

dt_classified = np.dtype([
    ('is_positive', 'bool'),
])

class ThresholdClassifier:
    """
    Simple classifier with threshold on first feature.
    the data with first feature larger than the specified threshold is regarded as positive.
    """
    def __init__(self, threshold):
        self.threshold = threshold

    def predict(self, X):
        return X[:,0] > self.threshold

class LinearClassifier2D:
    """
    Sample satisfies `a * X[:,0] + b * X[:,1] + c > 0` is regarded as positive.
    """
    def __init__(self, a,b,c):
        self.a = a
        self.b = b
        self.c = c

    def predict(self, X):
        return self.a * X[:,0] + self.b * X[:,1] + self.c > 0

def get_X_whole(wbc, normalizer_img=None, **kwargs):

    info = joblib.load(os.path.join(wbc.wholebrain_images.params["dst_basedir"], "info.pkl"))
    X_whole = []
    for src_pkl in info.keys():
        yname,xname = src_pkl.split("/")[-1].split(".")[0].split("_")
        if "FW" in src_pkl:
            cellstack = wbc.halfbrain_cells_FW.get_stack_by_xyname(xname=xname, yname=yname)
        else:
            cellstack = wbc.halfbrain_cells_RV.get_stack_by_xyname(xname=xname, yname=yname)

        data_scalemerged_stack = joblib.load(src_pkl)
        X = get_X_with_normalizer(cellstack.data_local, data_scalemerged_stack, normalizer_img, is_ave=wbc.is_ave, **kwargs)

        X_whole.append(X)
    X_whole = np.concatenate(X_whole)
    return X_whole

def get_data_scalemerged_whole(wbc, clf=None, normalizer_img=None, return_X=False, **kwargs):
    info = joblib.load(os.path.join(wbc.wholebrain_images.params["dst_basedir"], "info.pkl"))
    list_data_scalemerged = []
    if return_X:
        X_whole = []
    for src_pkl in info.keys():
        yname,xname = src_pkl.split("/")[-1].split(".")[0].split("_")
        if "FW" in src_pkl:
            cellstack = wbc.halfbrain_cells_FW.get_stack_by_xyname(xname=xname, yname=yname)
        else:
            cellstack = wbc.halfbrain_cells_RV.get_stack_by_xyname(xname=xname, yname=yname)
        data_scalemerged_stack = joblib.load(src_pkl)

        if clf:
            X = get_X_with_normalizer(cellstack.data_local, data_scalemerged_stack, normalizer_img, is_ave=wbc.is_ave, **kwargs)
            if X.shape[0] == 0:
                continue
            pred = clf.predict(X)
            list_data_scalemerged.append(data_scalemerged_stack[data_scalemerged_stack["is_valid"]][pred])
            if return_X:
                X_whole.append(X)
        else:
            list_data_scalemerged.append(data_scalemerged_stack[data_scalemerged_stack["is_valid"]])
            if return_X:
                X = get_X_with_normalizer(cellstack.data_local, data_scalemerged_stack, normalizer_img, is_ave=wbc.is_ave, **kwargs)
                X_whole.append(X)

    data_scalemerged_whole = np.concatenate(list_data_scalemerged)
    if not return_X:
        return data_scalemerged_whole
    else:
        X_whole = np.concatenate(X_whole)
        return data_scalemerged_whole, X_whole

def make_density_image(wbc, clf=None, dtype=np.uint16, normalizer_img=None, **kwargs):
    data_scalemerged_whole = get_data_scalemerged_whole(wbc, clf, normalizer_img, **kwargs)

    depth = int(np.floor(np.max(data_scalemerged_whole["scaled_z"])))
    height = int(np.floor(np.max(data_scalemerged_whole["scaled_y"])))
    width = int(np.floor(np.max(data_scalemerged_whole["scaled_x"])))
    density_img,_ = np.histogramdd(
        np.vstack([
            data_scalemerged_whole["scaled_z"],
            data_scalemerged_whole["scaled_y"],
            data_scalemerged_whole["scaled_x"]
        ]).T,
        bins=(depth, height, width),
        range=[(0,depth-1),(0,height-1),(0,width-1)]
    )
    return density_img.astype(dtype)

# Feature Vector
def get_X_with_normalizer(data_local, data_scalemerged, normalizer_img=None, **kwargs):

    data_local_valid = data_local[data_scalemerged["is_valid"]]

    if data_local_valid.shape[0] == 0:
        return np.empty((0,3), dtype=np.float32)

    X = get_X_3d(data_local_valid, **kwargs)

    if normalizer_img is not None:
        data_scalemerged_valid = data_scalemerged[data_scalemerged["is_valid"]]
        X[:, 0] -= np.log10(normalizer_img[
            np.clip(np.floor(data_scalemerged_valid["scaled_z"]).astype(int),
                    a_min=0,a_max=normalizer_img.shape[0]-1),
            np.clip(np.floor(data_scalemerged_valid["scaled_y"]).astype(int),
                    a_min=0,a_max=normalizer_img.shape[1]-1),
            np.clip(np.floor(data_scalemerged_valid["scaled_x"]).astype(int),
                    a_min=0,a_max=normalizer_img.shape[2]-1)
        ])
    return X

def get_X_3d(data,
             bias=np.array([0, 0, 0]),
             scale=np.array([1.0, 1.0, 1.]),
             bias_before_log=np.array([0.0,0.0,0.0]),
             is_ave=False):
    if not is_ave:
        _X = np.array([
            data["intensity"],
            data["structureness"],
            data["blobness"]
        ]).T
    else:
        _X = np.array([
            data["intensity"] / data["size"],
            data["structureness"],
            data["blobness"]
        ]).T
    X = np.nan_to_num((np.log10(_X + bias_before_log) + bias) * scale)
    return X

def predict_unsupervised(X, i_feature_maximize=1, n_components=3, **vargs):
    vbgmm = BayesianGaussianMixture(
        weight_concentration_prior_type="dirichlet_process",
        n_components=n_components,
        **vargs
    )
    vbgmm.fit(X)
    pred = vbgmm.predict(X)

    if i_feature_maximize is None:
        return pred
    else:
        i_pred_cluster = np.argmax(vbgmm.means_[:,i_feature_maximize])
        return pred == i_pred_cluster

def train_decision_tree(X, y, max_depth=2):
    pipe = Pipeline([
        ('pca', PCA()),
        ('tree', DecisionTreeClassifier(max_depth=max_depth))
    ])
    X_train,X_test,y_train,y_test = train_test_split(X, y, test_size=.2, shuffle=True)
    pipe.fit(X_train, y_train)
    print("Validation Score:{}".format(pipe.score(X_test, y_test)))
    pred = pipe.predict(X)
    return pipe, pred

def show_decision_tree(pipe, feature_names=["intensity","structureness","blobness"]):
    if isinstance(pipe, Pipeline):
        clf = pipe.named_steps['tree']
    elif isinstance(pipe, DecisionTreeClassifier):
        clf = pipe
    dot_data = export_graphviz(clf, out_file=None,
                               feature_names=feature_names,
                               filled=True, rounded=True,
                               special_characters=True)
    graph = graphviz.Source(dot_data)
    return graph

def plot_classification_result(feature_x, feature_y, pred):
    sns.scatterplot(feature_x, feature_y, hue=pred)
    if np.count_nonzero(pred == 0) > 1:
        sns.kdeplot(feature_x[pred==0],feature_y[pred==0], cmap="Blues", shade=True, shade_lowest=False)
    if np.count_nonzero(pred == 1) > 1:
        sns.kdeplot(feature_x[pred==1],feature_y[pred==1], cmap="Oranges", shade=True, shade_lowest=False)
    return

def plot_features_for_each_stacks(r, func_get_X, clf=None):
    i_xs = []
    i_ys = []
    x1s = []
    x2s = []
    preds=[]
    for i_y in range(len(r.stack_ys)):
        print(i_y)
        for i_x in range(len(r.stack_xs)):
            _stack = r.get_stack(i_xy=(i_x,i_y), verbose=False)
            if len(_stack.df) < 10000: continue
            _X_stack = func_get_X(_stack.df)
            if clf: _pred_stack = clf.predict(_X_stack)
            for i in range(0,len(_stack.df), 1000):
                i_xs.append(i_x)
                i_ys.append(i_y)
                x1s.append(_X_stack[i,0])
                x2s.append(_X_stack[i,1])
                if clf: preds.append(_pred_stack[i])

    df_plot = pd.DataFrame({
        "i_x":pd.Series(i_xs),
        "i_y":pd.Series(i_ys),
        "X1":pd.Series(x1s),
        "X2":pd.Series(x2s),
        "predicted":pd.Series(preds)
    })
    if not clf:
        g = sns.FacetGrid(df_plot, row="i_y", col="i_x",
                          height=2, aspect=1,)
        g.map(sns.kdeplot, "X1", "X2", shade=True, shade_lowest=False)
    else:
        g = sns.FacetGrid(df_plot, row="i_y", col="i_x",hue="predicted",
                      hue_kws={"cmap":["Blues", "Oranges"]},
                      height=2, aspect=1,)
        g.map(sns.kdeplot, "X1", "X2",)

def make_pred_img(df, pred, zlim=None, sigma=(1.0, 2.0, 2.0)):
    # zlim = (start_z, end_z) or None(:=all)
    # sigma = (sigma_z, sigma_y, sigma_x) or None(:=load from parameter file)
    if not zlim:
        z0 = 0
        depth = int(np.max(np.array(df["local_x"])))
    else:
        z0 = zlim[0]
        depth = zlim[1]-zlim[0]
    if not sigma:
        raise ValueError

    pred_img = np.zeros((depth,2160,2560), dtype=np.uint16)
    for i in np.nonzero(pred)[0]:
        z = int(np.array(df["local_z"])[i] - z0)
        if z < 0 or z >= depth: continue
        y = int(np.array(df["local_y"])[i])
        x = int(np.array(df["local_x"])[i])
        #pred_img[z-1:z+2, y-1:y+2, x-1:x+2] = 1000
        pred_img[z, y, x] = 5000
    pred_img = scipy.ndimage.filters.gaussian_filter(pred_img, sigma=sigma, truncate=2.)
    return pred_img

def save_classified_result_for_stack(dst_path, cellstack, clf):
    X = get_X_3d(cellstack.data_local)
    if X.shape[0] == 0:
        data_clf = np.zeros(0, dtype=dt_classified)
    else:
        data_clf = np.zeros(X.shape[0], dtype=dt_classified)
        data_clf["is_positive"] = clf.predict(X)
    joblib.dump(data_clf, dst_path, compress=3)
    return np.count_nonzero(data_clf["is_positive"])

def main():
    args = docopt(__doc__)

    with open(args["PARAM_FILE"]) as f:
        params = json.load(f)

    wbc = WholeBrainCells(params["MergeBrain_paramfile"])
    dst_basedir = params["dst_basedir"]
    dst_basedir_FW = os.path.join(dst_basedir,"FW")
    dst_basedir_RV = os.path.join(dst_basedir,"RV")
    if not os.path.exists(dst_basedir_FW):
        os.makedirs(dst_basedir_FW)
    if not os.path.exists(dst_basedir_RV):
        os.makedirs(dst_basedir_RV)

    # prepare feature vectors
    print("[*] preparing feature vectors...")
    X_whole = get_X_whole(wbc)
    print("Total candidates:", X_whole.shape[0])
    num_skip = params["num_skip_samples"]

    num_components = params["automatic_boundary"]["num_clusters"]
    i_feature_maximize = params["automatic_boundary"]["i_feature_maximize"]

    print("[*] running unsupervised clustering...")
    pred_un = predict_unsupervised(
        X_whole[::num_skip,:],
        n_components=num_components,
        i_feature_maximize=i_feature_maximize
    )

    if params["use_manual_boundary"]:
        print("[*] creating manual classifier...")
        a = int(params["manual_boundary"]["a"])
        b = int(params["manual_boundary"]["b"])
        c = int(params["manual_boundary"]["c"])
        clf = LinearClassifier2D(a,b,c)
        pred = clf.predict(X_whole[::num_skip])

    else:
        # train supervised decision tree classifier
        print("[*] training supervised classifier...")
        clf, pred = train_decision_tree(X_whole[::num_skip], pred_un)

    # save classifier
    clf_filename = os.path.join(
        params["dst_basedir"],
        "{}.pkl".format(params["classifier_name"]))
    joblib.dump(clf, clf_filename)
    print("[*] saved the classifier to {}".format(clf_filename))

    # save classified results & count total number of cells
    print("[*] saving classified results...")
    dict_num_cells = {}
    for xyname,cellstack in wbc.halfbrain_cells_FW.dict_stacks.items():
        if cellstack.is_dummy: continue
        dst_path = os.path.join(dst_basedir_FW, "{}_{}.pkl".format(xyname[1],xyname[0]))
        num_cells = save_classified_result_for_stack(dst_path, cellstack, clf)
        dict_num_cells[dst_path] = num_cells

    for xyname,cellstack in wbc.halfbrain_cells_RV.dict_stacks.items():
        if cellstack.is_dummy: continue
        dst_path = os.path.join(dst_basedir_RV, "{}_{}.pkl".format(xyname[1],xyname[0]))
        num_cells = save_classified_result_for_stack(dst_path, cellstack, clf)
        dict_num_cells[dst_path] = num_cells

    # save statistics
    joblib.dump(dict_num_cells, os.path.join(dst_basedir, "info.pkl"), compress=3)
    total_cells = sum(list(dict_num_cells.values()))
    print("Total cells:", total_cells)

    # make cell density image
    print("[*] making cell density image...")
    density_img = make_density_image(wbc, clf)
    img_filename = os.path.join(
        params["dst_basedir"],
        "{}.tif".format(params["density_img_name"]))
    tifffile.imsave(
        img_filename,
        density_img.astype(np.uint16)
    )
    print("[*] saved density image to {}".format(img_filename))

    fig = plt.figure(figsize=(10,5))
    fig.add_subplot(1,2,1)
    plot_classification_result(
        X_whole[::num_skip,0],
        X_whole[::num_skip,1],
        pred_un
    )
    plt.title("unsupervised clustering result")
    fig.add_subplot(1,2,2)
    plot_classification_result(
        X_whole[::num_skip,0],
        X_whole[::num_skip,1],
        pred
    )
    if params["use_manual_boundary"]:
        plt.title("manual classification result")
    else:
        plt.title("supervised classification result")
    plt.savefig(os.path.join(
        params["dst_basedir"],
        "feature_space_{}.png".format(params["classifier_name"])
    ))

    return

if __name__ == "__main__":
    main()


def printMenu():
    ''' Receives algebraic operation desired on two numbers from user '''
    while True:
        op = input("Enter operation:\n"
                   "1. Addition\n"
                   "2. Subtraction\n"
                   "3. Multiplication\n"
                   "4. Division\n") # menu of algebraic operations
        if op == '': # enter = quit
            break
        if op == '1' or op == '2' or op == '3' or op =='4': # if user entered a valid operation
            num_in1 = num_in2 = ''
            while not num_in1.isdigit(): # keep asking for first number until valid number given
                num_in1 = input("Enter first number: ")
            while not num_in2.isdigit(): # keep asking for second number until valid number given
                num_in2 = input("Enter second number: ")
            num1 = int(num_in1) # convert string inputs into integers
            num2 = int(num_in2)
            if op == '1': # add
                res = addition(num1, num2)
                print(f"{num1} + {num2} = {res}\n")
            elif op == '2': # subtract
                res = sub(num1, num2)
                print(f"{num1} - {num2} = {res}\n")
            elif op == '3': # multiply
                res = mult(num1, num2)
                print(f"{num1} * {num2} = {res}\n")
            elif op == '4': # divide
                res = div(num1, num2)
                if res != None:
                    print(f"{num1} / {num2} = {res}\n")
        else:
            print("Invalid operation chosen\n")

def addition(num1, num2):
    ''' Adds the two numbers '''
    return num1 + num2

def sub(num1, num2):
    ''' Subtracts second number from first number '''
    return num1 - num2

def mult(num1, num2):
    ''' Multiplies the two numbers '''
    return num1 * num2

def div(num1, num2):
    ''' Divides the first number by the second number '''
    if num2 == 0:
        print("Cannot divide by zero\n")
        return
    return num1 / num2

def main():
    printMenu()

if __name__ == '__main__':
    main()


'''
This is the app.py acting as a server
'''
import os
from flask import Flask, send_from_directory, json
from flask_socketio import SocketIO
from flask_cors import CORS
from flask_sqlalchemy import SQLAlchemy
from dotenv import load_dotenv, find_dotenv

load_dotenv(find_dotenv())

APP = Flask(__name__, static_folder='./build/static')

# Point SQLAlchemy to your Heroku database
APP.config['SQLALCHEMY_DATABASE_URI'] = os.getenv('DATABASE_URL')
# Gets rid of a warning
APP.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

DB = SQLAlchemy(APP)
# IMPORTANT: This must be AFTER creating db variable to prevent
# circular import issues

import models
DB.create_all()

CORS = CORS(APP, resources={r"/*": {"origins": "*"}})

SOCKETIO = SocketIO(APP,
                    cors_allowed_origins="*",
                    json=json,
                    manage_session=False)

LOGIN_NAME = []

@APP.route('/', defaults={"filename": "index.html"})
@APP.route('/<path:filename>')
def index(filename):
    '''
    This is a index function
    '''
    return send_from_directory('./build', filename)

# When a client connects from this Socket connection, this function is run
@SOCKETIO.on('connect')
def on_connect():
    '''
    This is a on_connect function
    '''
    print('User connected!')

@SOCKETIO.on('join')
def on_join(data):
    '''
    This is a on_join function
    '''# data is whatever arg you pass in your emit call on client
    print(str(data))
    users = add_user(data['users'])
    SOCKETIO.emit('user_list', {'users': users})

def add_user(username):
    '''
    This is a add_user function
    '''
    new_user = models.Person(username=username,
                             email='{0}@stuff.com'.format(username))
    DB.session.add(new_user)
    DB.session.commit()
    all_people = models.Person.query.all()
    users = []
    for person in all_people:
        users.append(person.username)
    return users

# When a client disconnects from this Socket connection, this function is run
@SOCKETIO.on('disconnect')
def on_disconnect():
    '''
    This is a on_disconnect function
    '''
    print('User disconnected!')

@SOCKETIO.on('login')
def on_login(data):
    '''
    This is a on_login function
    '''
    print(data)
    name = data['message']
    LOGIN_NAME.append(name)
    print(LOGIN_NAME)
    SOCKETIO.emit('login', LOGIN_NAME, broadcast=True, include_self=True)

# When a client emits the event 'chat' to the server, this function is run
# 'chat' is a custom event name that we just decided
@SOCKETIO.on('chat')
def on_chat(data):
    '''
    This is a on_chat function
    '''# data is whatever arg you pass in your emit call on client
    print(str(data))
    # This emits the 'chat' event from the server to all clients except for
    # the client that emmitted the event that triggered this function
    SOCKETIO.emit('chat', data, broadcast=True, include_self=False)

# Note we need to add this line so we can import app in the python shell
if __name__ == "__main__":
    # Note that we don't call app.run anymore. We call socketio.run with app arg
    SOCKETIO.run(
        APP,
        host=os.getenv('IP', '0.0.0.0'),
        port=8081 if os.getenv('C9_PORT') else int(os.getenv('PORT', '8081')),
    )


'''
数据增强，包括：随机水平翻转、随机垂直翻转、随机20度范围内的剪切、随机20%范围内的缩放、随机20%范围内的水平平移，随机20%范围内的垂直平移，
随机20度范围内的旋转。变换时，边界外像素点的填充采用“reflect”模式。图像均被缩放为256*256大小，像素值被调整到0-1之间。
'''

# this is import from Augmentor third-party package
# import Augmentor
#
# # create a new pipeline
# file_path = "C:\\liyu\\files\\tiff\\cells"
# p = Augmentor.Pipeline(file_path)

from Augmentor.Pipeline import *
import time

# create a new pipeline
file_path = "C:\\liyu\\files\\tiff\\newtest"
p = Pipeline(file_path)

# add operations to the pipeline

# 水平翻转
p.flip_left_right(probability=0.8)

# # 垂直翻转
# p.flip_top_bottom(probability=0.5)
#
# # 随机90， 180， 270 度旋转
# p.rotate_random_90(probability=0.75)
#
# # 随机20度内旋转，不变形, 四角填充黑色，图片大小不变
# p.rotate_without_crop(probability=0.5, max_left_rotation=20, max_right_rotation=20)
#
# # 随机20度内旋转，不变形, 四角填充黑色，图片大小调整
# p.rotate_without_crop(probability=0.5, max_left_rotation=20, max_right_rotation=20, expand=True)
#
# # 随机剪切, 中心不变
# p.crop_by_size(probability=0.5, width=100, height=100)
#
# # 随机剪切，中心变化，可调节剪切比例
# p.crop_random(probability=0.5, percentage_area=0.8)
#
# # 随机20%缩放，大小不变，图片缩小时以黑色填充
# p.zoom(probability=0.5, min_factor=0.8, max_factor=1.2)

# resize
p.resize(probability=1, width=256, height=256)

# execute and sample from the pipeline
time1 = time.time()

num_of_samples = 14200
p.sample(num_of_samples)

time2 = time.time()
print("time cost = " + str(time2-time1) + "s")


# -*- coding: utf-8 -*-

import sys
import os.path
from datetime import datetime

FILEPATH =os.path.expanduser('~')+"/.vocfile"

def create_vocable(eng, de):
    voc = {}
    voc["eng"]             = eng
    voc["de"]              = de
    voc["score"]           = 1
    voc["last_considered"] = datetime.today()
    return voc

def add_voc_to_file(vocable):
    # open file
    if os.path.isfile(FILEPATH):
        f = open(FILEPATH, "a")
    else:
        f = open(FILEPATH, "w+")
    # trim voc_args to first word
    if not has_duplicate(vocable["eng"]):
        # create formated string
        isFirstWord = True
        s = vocable["eng"]+":"
        for de in vocable["de"]:
            s = s+de+","
        s = s[0:-1]+':' # last comma is waste
        s += str(vocable["score"])+":"+vocable["last_considered"].strftime("%d/%m/%y")
        # write string
        f.write(s+"\n")
    else:
        print("Word already in list")
    f.close()

# checks if s is already in $vocfile
def has_duplicate(s):
    f = open(FILEPATH, "r")
    for line in f:
        # trim line to the first word
        if (s == line[0:line.index(':')]):
            f.close()
            return True
    f.close()
    return False

#
def check_input(input, vocable):
    vocable["last_considered"] = datetime.today()
    if input in vocable["de"]:
        vocable["score"] += 1
        return True
    else:
        vocable["score"] = 1
        return False

#
def parse_vocfile():
    vocable_list = []
    # open file
    if os.path.isfile(FILEPATH):
        vocfile = open(FILEPATH, "r")
    else:
        print("error: vocfile does not exist")
    for line in vocfile:
        tmp = line.split(':')
        voc = {}
        voc["eng"]             = tmp[0]
        voc["de"]              = tmp[1].split(',')
        voc["score"]           = int(tmp[2])
        # tmp[3][0:-2] to get rid off the "\n"
        voc["last_considered"] = datetime.strptime(tmp[3][0:-1], "%d/%m/%y")
        vocable_list.append(voc)
    return vocable_list

def get_test_vocables(vocable_list):
    vocables_to_test = []
    for voc in vocable_list:
        if voc["score"] == 1:
            vocables_to_test.append(voc)
        elif voc["score"] == 2 and (datetime.today() - datetime.strptime(voc["last_considered"], "%d/%m/%y")).days > 1:
            vocables_to_test.append(voc)
        elif voc["score"] == 3 and (datetime.today() - datetime.strptime(voc["last_considered"], "%d/%m/%y")).days > 7:
            vocables_to_test.append(voc)
        elif voc["score"] == 4 and (datetime.today() - datetime.strptime(voc["last_considered"], "%d/%m/%y")).days > 30:
            vocables_to_test.append(voc)
    return vocables_to_test

add_voc_to_file(create_vocable("jump", ['springen', 'hüpfen', 'emporschnellen']))
add_voc_to_file(create_vocable("run", ['rennen']))
add_voc_to_file(create_vocable("blame", ['beschämen']))

test_vocables = get_test_vocables(parse_vocfile())
for voc in test_vocables:
    user_input = input('translate: ' + voc["eng"] + ' - ')
    if check_input(user_input, voc):
        print("Prima")
    else:
        print("Idiot")


# #####################################################################################################################
#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.                                                 #
#                                                                                #
#  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance     #
#  with the License. You may obtain a copy of the License at                                                          #
#                                                                                #
#   http://www.apache.org/licenses/LICENSE-2.0                                                                        #
#                                                                                #
#  Unless required by applicable law or agreed to in writing, software distributed under the License is distributed   #
#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for  #
#  the specific language governing permissions and limitations under the License.                                     #
# #####################################################################################################################

from pathlib import Path
from typing import Dict

from aws_cdk.aws_lambda import Function
from aws_cdk.core import (
    Stack,
    CfnParameter,
    App,
    CfnCondition,
    Fn,
    CfnResource,
    Aspects,
    Duration,
    Aws,
    RemovalPolicy,
)

from demo.downloader import DemoDownloader
from demo.policies import DemoPolicies
from forecast.parameters import Parameters
from forecast.stack import NestedForecastStack
from interfaces import TemplateOptions, ConditionalResources
from interfaces.CfnNagSuppressAll import CfnNagSuppressAll
from solutions.cfn_nag import CfnNagSuppression
from solutions.mappings import Mappings
from solutions.metrics import Metrics
from stepfunctions.lambda_builder import LambdaBuilder

# These are the defaults for the CloudFormation parameters
FORECAST_NAME_DEFAULT = "nyctaxi_weather_auto"
FORECAST_CONFIG_DEFAULT = "s3://amazon-forecast-samples/automation_solution/demo-nyctaxi/forecast-defaults.yaml"
TTS_URL_DEFAULT = "s3://amazon-forecast-samples/automation_solution/demo-nyctaxi/nyctaxi_weather_auto.csv"
RTS_URL_DEFAULT = "s3://amazon-forecast-samples/automation_solution/demo-nyctaxi/nyctaxi_weather_auto.related.csv"
MD_URL_DEFAULT = "s3://amazon-forecast-samples/automation_solution/demo-nyctaxi/nyctaxi_weather_auto.metadata.csv"
FORECAST_STACK_DEFAULT = ""
CFN_RESOURCE_TYPE_URL_INFO = "Custom::UrlInfo"

class DemoStack(Stack):
    def __init__(self, app: App, id: str, **kwargs) -> None:
        super().__init__(app, id, **kwargs)

        template_options = TemplateOptions(
            self,
            id=id,
            description="(SO0123td) Improving Forecast Accuracy with Machine Learning %%VERSION%% - NYC Taxi Data Demo",
            filename="improving-forecast-accuracy-with-machine-learning-demo.template",
        )
        self.mappings = Mappings(self, "SO0123td")
        self.policies = DemoPolicies(self)

        re_http_https_s3 = (
            r"^https?://([-\w\.]+)+(:\d+)?(/([-\w/_\.]*(\?\S+)?)?)?$|^s3://(.*)/(.*)$"
        )
        re_http_https_s3_or_blank = r"^https?://([-\w\.]+)+(:\d+)?(/([-\w/_\.]*(\?\S+)?)?)?$|^s3://(.*)/(.*)$|^$"

        forecast_name = CfnParameter(
            self,
            id="ForecastName",
            type="String",
            description="Name of the forecast to create in the Amazon Forecast console",
            default=FORECAST_NAME_DEFAULT,
            allowed_pattern=r"^[a-zA-Z][a-zA-Z0-9_]*$",
            max_length=63,
            constraint_description="Forecast names must be less than 63 characters, start with a letter, and contain only alphanumeric characters or underscores",
        )

        forecast_defaults_url = CfnParameter(
            self,
            id="ForecastDefaultsUrl",
            type="String",
            description="URL (S3, HTTP or HTTPS) your forecast defaults file (usually named forecast-defaults.yaml)",
            allowed_pattern=re_http_https_s3,
            default=FORECAST_CONFIG_DEFAULT,
            constraint_description="Must be a valid s3/http/https address",
        )

        tts_url = CfnParameter(
            self,
            id="TargetTimeSeriesUrl",
            type="String",
            description="URL (S3, HTTP or HTTPS) for target time series data",
            allowed_pattern=re_http_https_s3,
            default=TTS_URL_DEFAULT,
            constraint_description="Must be a valid s3/http/https address",
        )

        rts_url = CfnParameter(
            self,
            id="RelatedTimeSeriesUrl",
            type="String",
            description="URL (S3, HTTP or HTTPS) for related time series data",
            allowed_pattern=re_http_https_s3_or_blank,
            default=RTS_URL_DEFAULT,
            constraint_description="Must be a valid s3/http/https address or blank",
        )

        md_url = CfnParameter(
            self,
            id="MetadataUrl",
            type="String",
            description="URL (S3, HTTP or HTTPS) for item metadata",
            allowed_pattern=re_http_https_s3_or_blank,
            default=MD_URL_DEFAULT,
            constraint_description="Must be a valid http/https address or blank",
        )

        forecast_stack_name = CfnParameter(
            self,
            id="ForecastStackName",
            type="String",
            description="Existing forecast stack name",
            default=FORECAST_STACK_DEFAULT,
            constraint_description="Must be a valid CloudFormation stack name or blank (if blank, this will deploy a new stack)",
        )

        # Conditions
        self.defaults_provided = CfnCondition(
            self,
            "ForecastDefaultsProvided",
            expression=Fn.condition_not(Fn.condition_equals(forecast_defaults_url, "")),
        )

        self.tts_provided = CfnCondition(
            self,
            "TTSProvided",
            expression=Fn.condition_not(Fn.condition_equals(tts_url, "")),
        )

        self.rts_provided = CfnCondition(
            self,
            "RTSProvided",
            expression=Fn.condition_not(Fn.condition_equals(rts_url, "")),
        )

        self.md_provided = CfnCondition(
            self,
            "MDProvided",
            expression=Fn.condition_not(Fn.condition_equals(md_url, "")),
        )

        self.stack_provided = CfnCondition(
            self,
            "ForecastStackNameProvided",
            expression=Fn.condition_not(Fn.condition_equals(forecast_stack_name, "")),
        )

        self.stack_not_provided = CfnCondition(
            self,
            "ForecastStackNameNotProvided",
            expression=Fn.condition_equals(forecast_stack_name, ""),
        )

        # Parameter Groups:
        template_options.add_parameter_group(
            label="Forecast Configuration",
            parameters=[forecast_name, forecast_defaults_url],
        )
        template_options.add_parameter_group(
            label="Datasets Configuration",
            parameters=[tts_url, rts_url, md_url],
        )
        template_options.add_parameter_group(
            label="Forecast Stack (Optional)",
            parameters=[forecast_stack_name],
        )
        child_parameters = Parameters(self, limited_parameters=True)

        # Parameter Labels:
        template_options.add_parameter_label(tts_url, "Target Time Series URL")
        template_options.add_parameter_label(
            rts_url, "Related Time Series URL (or blank)"
        )
        template_options.add_parameter_label(md_url, "Item Metadata URL (or blank)")
        template_options.add_parameter_label(
            forecast_defaults_url, "Forecast Defaults URL (or blank)"
        )
        template_options.add_parameter_label(
            forecast_stack_name,
            "If provided, use an existing Improving Forecast Accuracy with Machine Learning stack",
        )
        template_options.add_parameter_label(
            child_parameters.lambda_log_level, "CloudWatch Log Level"
        )
        child_parameters.add_template_options(template_options)

        # Build Required Lambda Functions
        functions: Dict[str, Function] = {}
        lambda_builder = LambdaBuilder(
            self,
            log_level=child_parameters.lambda_log_level.value_as_string,
            source_path=Path(__file__).parent.parent.parent.parent,
        )
        functions.update(
            lambda_builder.functions_for(
                name="CfnResource",
                base="lambdas/cloudformation_resources",
                handlers=[
                    "url_helper.handler",
                    "stack_outputs.handler",
                    "solution_metrics.handler",
                ],
                timeout=Duration.seconds(30),
            )
        )

        # stack outputs for forecast bucket
        functions["CfnResourceStackOutputs"].role.add_to_policy(
            self.policies.cloudformation_read(forecast_stack_name.value_as_string)
        )
        Aspects.of(functions["CfnResourceStackOutputs"]).add(
            ConditionalResources(self.stack_provided)
        )

        self.forecast_defaults_url_info = CfnResource(
            self,
            "ForecastDefaultsUrlInfo",
            type=CFN_RESOURCE_TYPE_URL_INFO,
            properties={
                "ServiceToken": functions["CfnResourceUrlHelper"].function_arn,
                "Url": forecast_defaults_url.value_as_string,
            },
        )

        self.tts_url_info = CfnResource(
            self,
            "TTSUrlInfo",
            type=CFN_RESOURCE_TYPE_URL_INFO,
            properties={
                "ServiceToken": functions["CfnResourceUrlHelper"].function_arn,
                "Url": tts_url.value_as_string,
            },
        )

        self.rts_url_info = CfnResource(
            self,
            "RTSUrlInfo",
            type=CFN_RESOURCE_TYPE_URL_INFO,
            properties={
                "ServiceToken": functions["CfnResourceUrlHelper"].function_arn,
                "Url": rts_url.value_as_string,
            },
        )
        Aspects.of(self.rts_url_info).add(ConditionalResources(self.rts_provided))

        self.md_url_info = CfnResource(
            self,
            "MDUrlInfo",
            type=CFN_RESOURCE_TYPE_URL_INFO,
            properties={
                "ServiceToken": functions["CfnResourceUrlHelper"].function_arn,
                "Url": md_url.value_as_string,
            },
        )
        Aspects.of(self.md_url_info).add(ConditionalResources(self.md_provided))

        # synthesize the main stack
        forecast_stack_cdk = NestedForecastStack(
            self, "ForecastStack", parameters=child_parameters.for_deployment()
        )

        forecast_stack_cdk.nested_stack_resource.cfn_options.condition = (
            self.stack_not_provided
        )
        forecast_stack_cdk.nested_stack_resource.apply_removal_policy(
            RemovalPolicy.RETAIN
        )
        forecast_stack_cdk.nested_stack_resource.override_logical_id("ForecastStack")

        # create or read information about an existing stack
        cfn_stack_info = CfnResource(
            self,
            "ForecastStackInfo",
            type="Custom::StackInfo",
            properties={
                "ServiceToken": functions["CfnResourceStackOutputs"].function_arn,
                "Stack": Fn.condition_if(
                    "ForecastStackNameProvided",
                    forecast_stack_name.value_as_string,
                    Aws.NO_VALUE,
                ).to_string(),
            },
        )
        Aspects.of(cfn_stack_info).add(ConditionalResources(self.stack_provided))

        # prepare the nested stack that performs the actual downloads
        downloader = DemoDownloader(
            self,
            "DemoDownloader",
            parameters={
                "DestinationBucket": Fn.condition_if(
                    "ForecastStackNameProvided",
                    cfn_stack_info.get_att("ForecastBucketName").to_string(),
                    Fn.condition_if(
                        "ForecastStackNameNotProvided",
                        forecast_stack_cdk.nested_stack_resource.get_att(
                            "Outputs.ForecastBucketName"
                        ).to_string(),
                        Aws.NO_VALUE,
                    ),
                ).to_string(),
                "ForecastName": forecast_name.value_as_string,
                "LambdaLogLevel": child_parameters.lambda_log_level.value_as_string,
                **{
                    f"{file}{info}": Fn.condition_if(
                        f"{file}Provided",
                        Fn.get_att(f"{file}UrlInfo", f"{info}").to_string(),
                        "",
                    ).to_string()
                    for file in ["TTS", "RTS", "MD", "ForecastDefaults"]
                    for info in ["Url", "Scheme", "Bucket", "Key"]
                },
            },
        )
        downloader.nested_stack_resource.override_logical_id("DemoDownloader")

        self.metrics = Metrics(
            self,
            "SolutionMetrics",
            metrics_function=functions["CfnResourceSolutionMetrics"],
            metrics={
                "Solution": self.mappings.solution_mapping.find_in_map("Data", "ID"),
                "Version": self.mappings.solution_mapping.find_in_map(
                    "Data", "Version"
                ),
                "Region": Aws.REGION,
            },
        )

        # aspects
        Aspects.of(self).add(
            CfnNagSuppressAll(
                suppress=[
                    CfnNagSuppression(
                        "W89", "Solution AWS Lambda Functions are not deployed to a VPC"
                    ),
                    CfnNagSuppression(
                        "W92",
                        "Solution AWS Lambda Functions do not require reserved concurrency",
                    ),
                ],
                resource_type="AWS::Lambda::Function",
            )
        )


import requests
import json

__version__ = 1.0
__author__ = 'Islam Bahnasy'
__email__ = 'islam@cybertechlab.com'
__url__  = 'http://cybertechlab.com'

class DiffbotMWException(Exception):
	''' DiffbotMW Exception class

	Code 401: Invalid Token
	Code 500: Invalid API or wrong URL

	'''
	def __init__(self, errCode, errMsg):
		self.errCode = errCode
		self.errMsg = errMsg

	def __str__(self):
		return ("<DiffbotMWException: %s %s>" % (self.errCode, self.errMsg))

def diffbot_mw(token, url, api, fields, version):
	''' Sends HTTP GET request to the DiffbotMW API and returns

	JSON object of the result in case of success or
	raise 'DiffbotMWException' withe error code and message

	token:   developer token for authentication
	url:     target web page
	api:     type of API like 'article', 'image'
	fields:	 specify desired output fields or '*' to return all fields
	version: API version

	example:
	from diffbot_mw import diffbot_mw
	json_obj = diffbot_mw('...', 'http://...', 'article',
				fields=['*'], version=2)

	'''

	payload = { 'token': token, 'url': url, 'fields': fields }
	response = requests.get("http://api.diffbot.com/v" + str(version) + "/" +
							str(api), params=payload)
	json_obj = response.json()

	if response.status_code != 200:
		raise DiffbotMWException(str(json_obj['errorCode']), \
								str(json_obj['error']))

	return json_obj


from rest_framework import mixins, status
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet

from fix_the_news.users import models

class CustomCreateModelMixin(GenericViewSet, mixins.CreateModelMixin):

    def create(self, request, *args, **kwargs):
        data = request.data.copy()
        data["user"] = self.get_user(request).id
        serializer = self.get_serializer(data=data)
        serializer.is_valid(raise_exception=True)
        self.perform_create(serializer)
        headers = self.get_success_headers(serializer.data)
        return Response(
            serializer.data,
            status=status.HTTP_201_CREATED,
            headers=headers,
        )

    def get_user(self, request):
        if request.user.is_authenticated:
            return request.user
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        if x_forwarded_for:
            ipaddress = x_forwarded_for.split(',')[0]
        else:
            ipaddress = request.META.get('REMOTE_ADDR')
        return models.User.get_or_create_anonymous_user(ipaddress)

class CustomListModelMixin(GenericViewSet, mixins.ListModelMixin):

    allowed_filters = []

    def get_queryset(self):
        filters = {
            key: self.request.query_params.get(key)
            for key
            in self.allowed_filters
            if self.request.query_params.get(key)
        }
        return super().get_queryset().filter(**filters)

class CustomCreateRetrieveListViewSet(CustomCreateModelMixin,
                                      CustomListModelMixin,
                                      mixins.RetrieveModelMixin):
    pass

class CustomModelViewSet(CustomCreateModelMixin,
                         CustomListModelMixin,
                         mixins.RetrieveModelMixin,
                         mixins.UpdateModelMixin,
                         mixins.DestroyModelMixin):
    pass


# Функция принимает аргументы DOMAIN(имя домена(можно без http), FROM((точно работает в годах)начало временного промежутка,
# которгого ведется поиск), TO(конец промежутка)
# Вывод в json-файле типа [['timestamp','original_url','html_length'], [...],[...], ...]

import requests

domain_name = 'sberbank.ru'
from_date = 2017
to_date = 2019

def web_archive(DOMAIN, FROM, TO):
    # Готовим запрос.
    # Ключи web archive: [["urlkey","timestamp","original","mimetype","statuscode","digest","length"],
    web_request = "https://web.archive.org/cdx/search/cdx?url=*.{}/*&output=json&from={}&to={}&fl=timestamp,original,length".format(
        DOMAIN, str(FROM), str(TO))
    response = None
    try:
        response = requests.get(web_request)

        if response:
            json_response = response.json()
            return json_response
        else:
            return "Ошибка выполнения запроса:\n", "Http статус:", response.status_code, "(", response.reason, ")"

    except:
        return "Запрос не удалось выполнить. Проверьте подключение к сети Интернет."

print(web_archive(domain_name, from_date, to_date))


from flask.globals import request

class ElementDispatcher(object):
    def __init__(self, event_dispatcher, type, implicit_return=True, logger=None):
        self.event_dispatcher = event_dispatcher
        self.type = type
        self.logger = logger

    def handle(self, subject, *args, **kwargs):
        if self.logger:
            self.logger.info("ElementDispatcher - dispatch 'element.%s' subject: %s" % (self.type, subject))

        event = self.event_dispatcher.dispatch('element.%s' % self.type, {
            self.type: subject,
            'args': args,
            'kwargs': kwargs
        })

        return event

class FlaskRequestElementDispatcher(ElementDispatcher):
    """
    This class is used as a proxy between Flask global request object and Element

    """
    def handle(self, *args, **kwargs):
        event = ElementDispatcher.handle(self, request, *args, **kwargs)

        if event.has('response'):
            return event.get('response')

class FlaskResponseElementDispatcher(ElementDispatcher):
    def handle(self, response):
        event = ElementDispatcher.handle(self, response)

        if event.has('response'):
            return event.get('response')


class Solution:
    def islandPerimeter(self, grid):
        """
        :type grid: List[List[int]]
        :rtype: int
        """
        sumraw = 0

        height = len(grid)
        width = len(grid[0])
        if height == 1:
            return 2 * grid[0].count(1) + 2
        for i in range (height):
            sumraw += grid[i].count(1)
        if width == 1:
            return grid.count([1]) * 2 + 2
        sumraw *= 4

        for row in range(1,height):
            for column in range(width):
                if grid[row][column] == 1 and grid[row - 1][column] == 1:
                        sumraw -= 2
        for column in range(1,width):
            for row in range(height):
                if grid[row][column] == 1 and grid[row][column - 1]:
                    sumraw -= 2
        return sumraw


import json
import time
from threading import Event, Thread

from cryptocollector.utils.utils import setup_logger
from cryptocollector.websocket.websocket_monitor import WebsocketMonitor

from websocket import create_connection

logger = setup_logger(name=__name__)

class WebsocketClient(Thread):
    """

    A websocket class.

    """

    def __init__(self, url, timeout=60):
        """

        Inputs:
        ------
        url : str
            The url of the websocket.
        timeout : int
            The websocket connection timeout in seconds.

        """

        super().__init__()
        self.url = url
        self.timeout = timeout
        self.stop = Event()
        self.name  = None

        self._count_msg = 0
        self._time_recv = None

        self.monitor = None
        self.ws = None

    def run(self):

        while not self.stop.is_set():
            self.__go()
        else:
            return

    def __go(self):

        try:
            self.connect()
            self.listen()
        except Exception as e:
            self.stop.set()
            logger.critical(repr(e))
            try:
                self.disconnect()
            except Exception as e:
                logger.critical(repr(e))

    def __stop(self):
        """

        Disconnect from websocket and stop monitor thread.

        """

        try:
            self.disconnect()
        except Exception as e:
            logger.critical(repr(e))

        try:
            self.monitor.join()
        except Exception as e:
            logger.critical(repr(e))

    def connect(self):
        """

        Connect to websocket.

        """

        logger.info(f'{self.name} is connecting to: {self.url}')

        self.ws = create_connection(self.url, timeout=self.timeout)

        self.on_open()

    def listen(self):
        """

        Receive new messages from websocket.

        """

        self.monitor = WebsocketMonitor(ws=self.ws,
                                        time_recv=self._time_recv,
                                        stop=self.stop)
        self.monitor.daemon = True
        self.monitor.start()

        while not self.stop.is_set():

            try:

                data = self.ws.recv()
                self._time_recv = time.time()
                msg = json.loads(data)
                self.on_message(msg)
                self._count_msg += 1

                if self._count_msg % 100 == 0:
                    logger.info(f'Received {self._count_msg} messages.')

            except Exception as e:

                self.stop.set()
                self.on_error(e)
                logger.critical(repr(e))
                self.monitor.join()

    def disconnect(self):
        """

        Disconnect from websocket.

        """

        if self.ws:
            self.ws.close()

        logger.warn('Disconnected.')

    def on_open(self):
        """

        Called upon initialization of connection.

        Notes
        -----
        In the derived class this method
        should send a subscription message to
        the exchange.

        """

        pass

    def on_message(self, msg):
        """

        Called upon receipt of message

        Inputs
        ------
        msg : dict
            The message from the exchange.

        Notes
        -----
        In the derived class this method should
        parse the message and save it to storage.

        """

        pass

    def on_error(self, error):
        """

        Called when error occurs.

        Notes
        -----
        After error disconnect from websocket.

        """

        self.disconnect()

    def on_close(self):
        """

        Called when the websocket connection closes.

        """

        logger.warn('Websocket closed.')


from pathlib import Path
from time import ctime
from zipfile import ZipFile

path = Path("test.txt")
path.write_text("data")
print(path.read_text())

print(ctime(path.stat().st_mtime))

for path in Path().rglob('*.*'):  # get all files in a path
    print(path)

#zip file
with ZipFile("zipfile.zip", 'w') as zip:
    for path in Path().rglob('*.*'):  # zip all files in the current path
        zip.write(path)

with ZipFile("zipfile.zip") as zip:
    print(zip.namelist())

#csv
import csv

with open("data.csv",'w') as file:
    writer = csv.writer(file)
    writer.writerow(["id","price"])
    writer.writerow([1,3])
    writer.writerow([2,33])
    writer.writerow(["3","price"])

with open("data.csv") as file:
    reader = csv.reader(file)
    #print(list(reader)) iterator reader has functionality to move next
    for row in reader:
        print(row)


import requests
from bs4 import BeautifulSoup

page = requests.get("https://www.bilibili.com")
soup = BeautifulSoup(page.content, 'html.parser')

top_items = []

products = soup.select('div.info-box')
for elem in products:
    title = elem.select('div.info > p.title')[0].text
    review_label = elem.select('p.play')[0].text
    info = {
        "title": title.strip(),
        "review": review_label.strip()
    }
    top_items.append(info)

print(top_items)


import argparse
import matplotlib
matplotlib.use('agg')
import csv
import json
import multiprocessing as mp
import os
import random
import re
import sys
from functools import partial
from operator import attrgetter, itemgetter
import networkx as nx
import numpy as np
import pandas as pd
import time
from sofa_aisi import *
from sofa_common import *
from sofa_config import *
from sofa_hsg import *
from sofa_print import *
import grpc
import potato_pb2
import potato_pb2_grpc
import socket

# input: pfv(performance feature vector), Pandas.DataFrame
# output: hint, docker_image
def get_hint(features):

    if len(features) > 0:
        pfv = potato_pb2.PerformanceFeatureVector()
        for i in range(len(features)):
            name = features.iloc[i]['name']
            value = features.iloc[i]['value']
            print('%s%s%s' % (str(i).ljust(10), name.ljust(30), ('%.3lf'%value).ljust(20)))
            pfv.name.append(name)
            pfv.value.append(value)

        print('Wait for response from POTATO server...')
        myhostname = socket.gethostname()
        channel = grpc.insecure_channel('localhost:50051')
        stub = potato_pb2_grpc.HintStub(channel)
        request = potato_pb2.HintRequest( hostname = myhostname,
                                          pfv = pfv)
        response = stub.Hint(request)
        hint = response.hint
        docker_image = response.docker_image
    else:
        hint = 'There is no pfv to get hints.'
        docker_image = 'NA'

    print(hint)
    print(docker_image)
    return hint, docker_image

def dynamic_top_down(logdir, cfg, df_mpstat, df_cpu, df_gpu, features):
    print_title("Dynamic Top-Down Analysis")

    total_elapsed_time = {'usr':0, 'sys':0, 'gpu':0, 'iow':0}
    elapsed_time_ratio = {'usr':0, 'sys':0, 'gpu':0, 'iow':0}

    if len(df_mpstat) == 0 or len(df_cpu) == 0:
        print_warning('no mpstat and perf traces!')
        return features

    t_begin = df_mpstat.iloc[0]['timestamp']
    t_end = df_mpstat.iloc[-1]['timestamp']

    t = t_begin
    while t < t_end:
        window_begin = t - 0.1
        window_end = t
        t = t + 0.1

        if df_cpu.iloc[0].timestamp > window_end:
            continue
        cond1 = (df_cpu['timestamp'] > window_begin)
        cond2 = (df_cpu['timestamp'] <= window_end)
        df_cpu_interval = df_cpu[ cond1 & cond2 ]

        cond1 = (df_gpu['timestamp'] > window_begin)
        cond2 = (df_gpu['timestamp'] <= window_end)
        df_gpu_interval = df_gpu[ cond1 & cond2 ]

        cond1 = (df_mpstat['timestamp'] > window_begin)
        cond2 = (df_mpstat['timestamp'] <= window_end)
        df_mpstat_interval = df_mpstat[ cond1 & cond2 ]
        mp_usr = []
        mp_sys = []
        mp_iow = []
        for i in range(len(df_mpstat_interval)):
            ratios = df_mpstat_interval.iloc[i]['name'].split(':')[1].split('|')
            #print(ratios)
            mp_usr.append(0.1*int(ratios[1])/100.0)
            mp_sys.append(0.1*int(ratios[2])/100.0)
            mp_iow.append(0.1*int(ratios[4])/100.0)
        mp_usr = np.asarray(mp_usr)
        mp_sys = np.asarray(mp_sys)
        mp_iow = np.asarray(mp_iow)

        elapsed_time = {'usr':0, 'sys':0, 'gpu':0, 'iow':0}

        if len(df_mpstat_interval) > 0:
            elapsed_time['usr'] = mp_usr.max()
            elapsed_time['sys'] = mp_sys.max()
            elapsed_time['gpu'] = df_gpu_interval['duration'].sum()
            elapsed_time['iow'] = mp_iow.max()
            dominator = max(elapsed_time, key=elapsed_time.get)
            if elapsed_time['gpu'] > 0 :
                dominator = 'gpu'
            total_elapsed_time[dominator] = total_elapsed_time[dominator] + 0.1

    total_all_elapsed_time = sum(total_elapsed_time.values())
    if total_all_elapsed_time > 0 :
        elapsed_time_ratio['usr'] = 100 * total_elapsed_time['usr'] / total_all_elapsed_time
        elapsed_time_ratio['sys'] = 100 * total_elapsed_time['sys'] / total_all_elapsed_time
        elapsed_time_ratio['gpu'] = 100 * total_elapsed_time['gpu'] / total_all_elapsed_time
        elapsed_time_ratio['iow'] = 100 * total_elapsed_time['iow'] / total_all_elapsed_time
        print('Elapsed Time = %.1lf ' % total_all_elapsed_time)
        print('USR = %.1lf %%' % elapsed_time_ratio['usr'])
        print('SYS = %.1lf %%' % elapsed_time_ratio['sys'])
        print('GPU = %.1lf %%' % elapsed_time_ratio['gpu'])
        print('IOW = %.1lf %%' % elapsed_time_ratio['iow'])
        df = pd.DataFrame({'name':['elapsed_usr_time_ratio', 'elapsed_sys_time_ratio', 'elapsed_gpu_time_ratio', 'elapsed_iow_time_ratio'],
                        'value':[elapsed_time_ratio['usr'],elapsed_time_ratio['sys'],elapsed_time_ratio['gpu'],elapsed_time_ratio['iow'] ] },
                        columns=['name','value'])
        features = pd.concat([features, df])

    return features

def payload_sum(df):
    print((len(df)))

class Event:

    def __init__(self, name, ttype, timestamp, duration):
        self.name = name
        self.ttype = ttype  # 0 for begin, 1 for end
        self.timestamp = timestamp
        self.duration = duration

    def __repr__(self):
        return repr((self.name, self.ttype, self.timestamp, self.duration))

def nvsmi_profile(logdir, cfg, df_nvsmi, features):
    print_title("SM & MEM Profiling")

    if len(df_nvsmi) > 0 :
        sm_start = df_nvsmi.iloc[0].timestamp
        sm_end = df_nvsmi.iloc[-1].timestamp
        SM_time = sm_end - sm_start
        result = df_nvsmi.groupby(['deviceId','event'])['duration'].mean()
        result = result.astype(int)
        print(result)

        gpu_sm_util = df_nvsmi.groupby(['event'])['duration'].mean()[0]
        gpu_mem_util = df_nvsmi.groupby(['event'])['duration'].mean()[1]
        print('Average SM Utilization (%): ', int(gpu_sm_util))
        print('Average MEM Utilization (%): ', int(gpu_mem_util))
        print('Active GPU Time (s): %.3lf' % (SM_time * gpu_sm_util/100.0))
        df = pd.DataFrame({'name':['gpu_sm_util', 'gpu_mem_util'],
                        'value':[gpu_sm_util, gpu_mem_util] },
                        columns=['name','value'])
        features = pd.concat([features, df])

    return features

def gpu_profile(logdir, cfg, df_gpu, features):
    print_title("GPU Profiling")
    print('Per-GPU time (s):')
    groups = df_gpu.groupby("deviceId")["duration"]
    gpu_time = 0
    for key, item in groups:
        gpuid = int(float(key))
        per_gpu_time = groups.get_group(key).sum()
        print("[%d]: %lf" % (gpuid, per_gpu_time))
        gpu_time = gpu_time + per_gpu_time
    n_gpus = len(groups)
    print(("Total GPU time of all GPUs (s) = %.3lf" % gpu_time))

    kernel_time = 0
    grouped_df = df_gpu.groupby("copyKind")["duration"]
    for key, item in grouped_df:
        if key == 0:
            kernel_time = grouped_df.get_group(key).sum()

    nccl_time = 0
    grouped_df = df_gpu.groupby("name")["duration"]
    for key, item in grouped_df:
        #print("[%s]: %lf" % (key, grouped_df.get_group(key).sum()))
        if key.find("nccl") != -1:
            nccl_time = nccl_time + grouped_df.get_group(key).sum()

    features = comm_profile(logdir, cfg, df_gpu, features)

    get_top_k_events(df_gpu, 10)
    df = pd.DataFrame({'name':['gpu_time', 'num_gpus', 'kernel_time', 'nccl_time'],
                        'value':[gpu_time, n_gpus, kernel_time, nccl_time] },
                        columns=['name','value'])
    features = pd.concat([features, df])
    return features

def net_profile(logdir, cfg, df, features):
    print_title("Network Profiling:")
    grouped_df = df.groupby("name")["duration"]
    net_time = 0
    n_packets = 0
    for key, item in grouped_df:
        #print("[%s]: %lf" % (key, grouped_df.get_group(key).sum()))
        if key.find("network:tcp:") != -1:
            net_time = net_time + grouped_df.get_group(key).sum()
            n_packets = n_packets + 1
    print(("total network time (s) = %.3lf" % net_time))
    print(("total amount of network packets  = %d" % n_packets))
    df = pd.DataFrame({'name':['net_time'],
                        'value':[net_time] },
                        columns=['name','value'])
    features = pd.concat([features, df])
    return features

def cpu_profile(logdir, cfg, df):
    print_title("CPU Profiling:")
    print('elapsed_time (s) = %.6lf' % cfg.elapsed_time)
    grouped_df = df.groupby("deviceId")["duration"]
    total_exec_time = 0
    for key, item in grouped_df:
        if cfg.verbose:
            print(("[%d]: %lf" % (key, grouped_df.get_group(key).sum())))
        total_exec_time = total_exec_time + grouped_df.get_group(key).sum()

    print("total execution time (s) = %.3lf" % total_exec_time)

    cpu_detail_profile_df = df[['timestamp','duration','name']]
    cpu_detail_profile_df = cpu_detail_profile_df.sort_values(by=['duration'], ascending=False)
    cpu_detail_profile_df['ratio(%)'] = cpu_detail_profile_df['duration']/total_exec_time * 100
    cpu_detail_profile_df = cpu_detail_profile_df[['timestamp','ratio(%)','duration','name']]
    print(cpu_detail_profile_df[:20].to_string(index=False))

def vmstat_profile(logdir, cfg, df, features):
    print_title("VMSTAT Profiling:")
    df_name = df['name']

    vmstat_fieldnames = []
    fields = df_name.iloc[0].split('|')
    for field in fields:
       vmstat_fieldnames.append(field.split('=')[0])

    records = []
    for name in df_name:
        fields = name.split('|')
        row = []
        for field in fields:
            row.append(int(field.split('=')[1]))
        records.append(row)

    #1542188143.006416,-1,0.000010,-1,-1,-1,-1,-1,-1,-1,-1,r=1|b=0|sw=0|fr=12566580|bu=2140|ca=17433464|si=0|so=0|bi=0|bo=3|in=3|cs=2|usr=0|sys=0|idl=100|wa=0|st=0,-1
    vmstat_traces = pd.DataFrame(records)
    vmstat_traces.columns = vmstat_fieldnames

    vm_bi = vmstat_traces['bi'].mean()
    vm_bo = vmstat_traces['bo'].mean()
    vm_cs = vmstat_traces['cs'].mean()
    vm_in = vmstat_traces['in'].mean()
    print('sum of vmstat bi: ', vm_cs)
    print('sum of vmstat bo: ', vm_in)
    print('sum of vmstat cs: ', vm_bi)
    print('sum of vmstat in: ', vm_bo)

    df_feature = pd.DataFrame({ 'name':['vm_bi', 'vm_bo', 'vm_cs', 'vm_in' ],
                        'value':[vm_bi, vm_bo, vm_cs, vm_in] },
                        columns=['name','value'])
    features = pd.concat([features, df_feature])

    return features

def mpstat_profile(logdir, cfg, df, features):
    print_title("MPSTAT Profiling:")
    num_cores = int(df['deviceId'].max() + 1)
    df_summary = pd.DataFrame( np.zeros((num_cores,5)), columns=['USR','SYS','IDL','IOW','IRQ'])
    for i in range(len(df)):
        dt = df.iloc[i]['duration']
        core = int(df.iloc[i]['deviceId'])
        fields = df.loc[i,'name'].split('|')
        r_usr = float(fields[5])
        r_sys = float(fields[6])
        r_idl = float(fields[7])
        r_iow = float(fields[8])
        r_irq = float(fields[9])
        if r_idl == 100:
            dt_all = 0.1
        else:
            dt_all = dt/((100-r_idl)/100.0)
        t_usr = dt_all * r_usr/100.0
        t_sys = dt_all * r_sys/100.0
        t_idl = dt_all * r_idl/100.0
        t_iow = dt_all * r_iow/100.0
        t_irq = dt_all * r_irq/100.0
        df_summary.iloc[core]['USR'] = df_summary.iloc[core]['USR'] + t_usr
        df_summary.iloc[core]['SYS'] = df_summary.iloc[core]['SYS'] + t_sys
        df_summary.iloc[core]['IDL'] = df_summary.iloc[core]['IDL'] + t_idl
        df_summary.iloc[core]['IOW'] = df_summary.iloc[core]['IOW'] + t_iow
        df_summary.iloc[core]['IRQ'] = df_summary.iloc[core]['IRQ'] + t_irq

    print('CPU Utilization (%):')
    print('core\tUSR\tSYS\tIDL\tIOW\tIRQ')
    for i in range(len(df_summary)):
        t_sum = df_summary.iloc[i].sum()
        print('%3d\t%3d\t%3d\t%3d\t%3d\t%3d'%(i,int(100.0*df_summary.iloc[i]['USR']/t_sum),
                                                int(100.0*df_summary.iloc[i]['SYS']/t_sum),
                                                int(100.0*df_summary.iloc[i]['IDL']/t_sum),
                                                int(100.0*df_summary.iloc[i]['IOW']/t_sum),
                                                int(100.0*df_summary.iloc[i]['IRQ']/t_sum) ))
    print('CPU Time (s):')
    print('core\tUSR\tSYS\tIDL\tIOW\tIRQ')
    for i in range(len(df_summary)):
        t_sum = df_summary.iloc[i].sum()
        print('%3d\t%.2lf\t%.2lf\t%.2lf\t%.2lf\t%.2lf'%(i,
                                                df_summary.iloc[i]['USR'],
                                                df_summary.iloc[i]['SYS'],
                                                df_summary.iloc[i]['IDL'],
                                                df_summary.iloc[i]['IOW'],
                                                df_summary.iloc[i]['IRQ'] ))

    total_cpu_time = df_summary[['USR','SYS','IRQ']].sum().sum()
    print('Active CPU Time (s): %.3lf' % total_cpu_time)
    cpu_util = int(100*total_cpu_time / (num_cores*cfg.elapsed_time))
    print('Active CPU ratio (%%): %3d' % cpu_util)
    df_feature = pd.DataFrame({ 'name':['num_cores', 'cpu_util'],
                        'value':[num_cores, cpu_util] },
                        columns=['name','value'])
    features = pd.concat([features, df_feature])
    return features

def sofa_analyze(cfg):
    filein = []
    df_cpu = pd.DataFrame([], columns=cfg.columns)
    df_gpu = pd.DataFrame([], columns=cfg.columns)
    df_net = pd.DataFrame([], columns=cfg.columns)
    df_mpstat = pd.DataFrame([], columns=cfg.columns)
    df_vmstat = pd.DataFrame([], columns=cfg.columns)
    iter_summary = None
    logdir = cfg.logdir

    with open(logdir+'/misc.txt') as f:
        lines = f.readlines()
        elapsed_time = float(lines[0].split()[1])
        vcores = int(lines[2].split()[1])
        cfg.elapsed_time = float(lines[0].split()[1])

    filein_gpu = logdir + "gputrace.csv"
    filein_cpu = logdir + "cputrace.csv"
    filein_net = logdir + "nettrace.csv"
    filein_vmstat = logdir + "vmstat.csv"
    filein_mpstat = logdir + "mpstat.csv"
    filein_strace = logdir + "strace.csv"
    filein_nvsmi = logdir + "nvsmi_trace.csv"

    if os.path.isfile('%s/nvlink_topo.txt' % logdir):

        with open(logdir + 'nvlink_topo.txt') as f:
            lines = f.readlines()
            if len(lines) > 0:
                title = lines[0]
                num_gpus = 1
                for word in title.split():
                    if re.match(r'GPU', word) != None :
                       num_gpus = num_gpus + 1
                print_info(cfg,'# of GPUs: ' + str(num_gpus) )
                edges = []
                if len(lines) >= num_gpus+1:
                    for i in range(num_gpus):
                        connections = lines[1+i].split()
                        for j in range(len(connections)):
                            if connections[j] == 'NV1' or connections[j] == 'NV2':
                                edges.append((i,j-1))
                                #print('%d connects to %d' % (i, j-1))

                    ring_found = False
                    G = nx.DiGraph(edges)
                    # Try to find ring with its length of num_gpus
                    for cycle in nx.simple_cycles(G):
                        if len(cycle) == num_gpus:
                            print(("One of the recommended ring having length of %d" % len(cycle) ))
                            ring_found = True
                            os.system("mkdir -p sofalog/sofa_hints/")
                            xring_order = ','.join(map(str, cycle))
                            with open("sofalog/sofa_hints/xring_order.txt", "w") as f:
                                f.write('export CUDA_VISIBLE_DEVICES=' + xring_order)
                            break

                    # Try to find ring with its length of num_gpus/2
                    if not ring_found:
                        for cycle in nx.simple_cycles(G):
                            if len(cycle) == num_gpus/2:
                                print(("One of the recommended ring having length of %d" % len(cycle) ))
                                ring_found = True
                                os.system("mkdir -p sofalog/sofa_hints/")
                                xring_order = ','.join(map(str, cycle))
                                with open("sofalog/sofa_hints/xring_order.txt", "w") as f:
                                    f.write('export CUDA_VISIBLE_DEVICES=' + xring_order)
                                break
    # Construct Performance Features
    features = pd.DataFrame({'name':['elapsed_time'], 'value':[cfg.elapsed_time]}, columns=['name','value'])

    try:
        df_cpu = pd.read_csv(filein_cpu)
        cpu_profile(logdir, cfg, df_cpu)
    except IOError as e:
        df_cpu = pd.DataFrame([], columns=cfg.columns)
        print_warning("%s is not found" % filein_cpu)

    try:
        df_strace = pd.read_csv(filein_strace)
    except IOError as e:
        df_strace = pd.DataFrame([], columns=cfg.columns)
        print_warning("%s is not found" % filein_strace)

    try:
        df_net = pd.read_csv(filein_net)
        features = net_profile(logdir, cfg, df_net, features)
    except IOError as e:
        df_net = pd.DataFrame([], columns=cfg.columns)
        print_warning("%s is not found" % filein_net)

    try:
        df_vmstat = pd.read_csv(filein_vmstat)
        features = vmstat_profile(logdir, cfg, df_vmstat, features)
    except IOError as e:
        df_vmstat = pd.DataFrame([], columns=cfg.columns)
        print_warning("%s is not found" % filein_vmstat)

    try:
        df_mpstat = pd.read_csv(filein_mpstat)
        features = mpstat_profile(logdir, cfg, df_mpstat, features)
    except IOError as e:
        df_mpstat = pd.DataFrame([], columns=cfg.columns)
        print_warning("%s is not found" % filein_mpstat)

    try:
        df_nvsmi = pd.read_csv(filein_nvsmi)
        features = nvsmi_profile(logdir, cfg, df_nvsmi, features)
    except IOError:
        print_warning("nvsmi_trace.csv is not found")

    try:
        df_gpu = pd.read_csv(filein_gpu)
        features = gpu_profile(logdir, cfg, df_gpu, features)
    except IOError:
        df_gpu = pd.DataFrame([], columns=cfg.columns)
        print_warning("%s is not found. If there is no need to profile GPU, just ignore it." % filein_gpu)

    try:
        features = dynamic_top_down(logdir, cfg, df_mpstat, df_cpu, df_gpu, features)
    except IOError as e:
        print_warning("Some files are not found, which are needed for dynamic_top_down analysis")

    if cfg.enable_aisi:
        selected_pattern, iter_summary = sofa_aisi(logdir, cfg, df_cpu, df_gpu, df_strace, df_mpstat)

    print_title('Final Performance Features')
    print('%s%s%s' % ('ID'.ljust(10),'Feature'.ljust(30),'Value'.ljust(20)) )

    for i in range(len(features)):
        name = features.iloc[i]['name']
        value = features.iloc[i]['value']
        print('%s%s%s' % (str(i).ljust(10), name.ljust(30), ('%.3lf'%value).ljust(20)))

    if cfg.potato_server:
        print_title('POTATO Feedback')
        hint, docker_image = get_hint(features)
        print('Optimization hints: ' + hint)
        print('Tag of optimal image recommended from POTATO: ' + highlight(docker_image))
        print('Please re-launch KubeFlow Jupyter-notebook with the new tag.')

    print('\n\n')


#!/usr/bin/env python3

import argparse
import csv
import os
import re

def parse_args():
    parser = argparse.ArgumentParser(description="Aggregate scalability results")
    parser.add_argument("-i", "--input-log-dir", type=str, required=True,
                        help="input log directory")

    parser.add_argument("-o", "--output-file", type=str, required=True,
                        help="output file name")

    return parser.parse_args()

def parse_headers(dir_name):
    headers = []

    for s in dir_name.split("__"):
        parts = s.split("@")
        headers.append(parts[0])

    return headers

def parse_values(dir_name):
    values = []

    for s in dir_name.split("__"):
        parts = s.split("@")
        values.append(parts[1])

    return values

def aggregate_measurements(log_dir):
    headers = []
    rows = []
    with os.scandir(log_dir) as entries:
        for entry in entries:
            dir_name = entry.name
            if not dir_name.startswith('.') and entry.is_dir():
                if len(headers) == 0:
                    headers = parse_headers(dir_name) + ["throughput"]

                fname = os.path.join(log_dir, dir_name, "client.log")

                with open(fname, "r") as f:
                    match = re.search(r"^.*Throughput\:\s([0-9\.]+)", f.read(), re.MULTILINE)
                    tput = match.group(1)
                    rows.append(parse_values(dir_name) + [tput])

        return headers, rows

def write_measurements(fname, headers, rows):
    with open(fname, "w+") as f:
        writer = csv.writer(f)
        writer.writerow(headers)

        for row in rows:
            writer.writerow(row)

def main():
    args = parse_args()

    headers, rows = aggregate_measurements(args.input_log_dir)
    write_measurements(args.output_file, headers, rows)

if __name__ == "__main__":
    main()


## Read input as specified in the question.
## Print output as specified in the question.
def SecondLargest(arr, n):
    if n <= 1:
        return -2147483648
    else:
        count = 0
        ele = arr[0]
        for i in range(n):
            if arr[i] == ele:
                count += 1
        if count == n:
            return -2147483648
        # print(n)
        else:
            Largest = -999999
            SecondLargest = -999999
            for i in range(n):
                if arr[i] > Largest:
                    SecondLargest = Largest
                    Largest = arr[i]
                elif arr[i] > SecondLargest and arr[i] != Largest:
                    SecondLargest = arr[i]
    return SecondLargest

n = int(input())
arr = [int(x) for x in input().split()]
value = SecondLargest(arr, n)
print(value)


import copy
import json
from easyquant import exceptions
from easyquant.policy.indicator import RealTimeIndicator
from playground.top_5_profit_ratio import get_top_5_profit_ratio_stocks

class Rule(object):
    def __init__(self, name, get_val_func, operator, indicator):
        self.name = name
        self.indicator = indicator
        self.operator = operator
        self.get_val_func = get_val_func

    def filter(self, stocks):
        result = {}
        for code, data in stocks.items():
            try:
                if self.operator(self.get_val_func(data),
                                 self.indicator.get_compare_val(code)):
                    if not data.get('rule_results', None):
                        data['rule_results'] = self.indicator.get_all_val(code)
                    else:
                        data['rule_results'].update(
                            self.indicator.get_all_val(code))
                    result[code] = data
            except KeyError:
                pass
            except exceptions.StockValueZero:
                pass
        return result

class SelectedCodesRule(Rule):
    def __init__(self, name, codes):
        super(SelectedCodesRule, self).__init__(name, None, None, None)
        self.codes = codes

    def filter(self, stocks):
        result = {}
        for code, data in stocks.items():
            if code in self.codes:
                result[code] = data
        return result

class TopProfitRatioRule(Rule):
    def __init__(self, name):
        super(TopProfitRatioRule, self).__init__(name, None, None, None)
        self.top_5_stocks = get_top_5_profit_ratio_stocks()

    def filter(self, stocks):
        result = {}
        top_5_codes = [x['code'] for x in self.top_5_stocks]
        for code, data in stocks.items():
            if code in top_5_codes:
                for x in self.top_5_stocks:
                    if code == x['code']:
                        rule_result = copy.deepcopy(x)
                        rule_result.pop('code')
                        rule_result.pop('price')
                        rdata = json.loads(rule_result.pop('data'))
                        rule_result['report_date'] = \
                            list(rdata['report_date'].values())
                        rule_result['shares'] = \
                            list(rdata['shares'].values())
                        rule_result['divi'] = \
                            list(rdata['divi'].values())
                        data['rule_results'] = rule_result
                result[code] = data
        return result

if __name__ == "__main__":
    from easyquant.policy.indicator import ContinuousRedDaysIndicator
    from easyquant.policy.indicator import EdgeIndicator
    from easyquant.policy.indicator import StopLossIndicator
    from operator import gt, lt
    import tushare as ts
    hist = ts.get_hist_data("002597", start="2017-10-10", end="2017-11-17")
    cindicator = ContinuousRedDaysIndicator('60reddays', 2, 10)
    eindicator = EdgeIndicator('highest', 'high', 10)
    sindicator = StopLossIndicator('stoploss', {'002597': 31})
    cindicator.calculate("002597", hist)
    eindicator.calculate("002597", hist)
    hist["002597"] = 0.0
    sindicator.calculate("002597", hist)
    rule1 = Rule('rule1', lambda x: 4, lt, cindicator)
    rule2 = Rule('rule2', lambda x: x['now'], gt, eindicator)
    rule3 = Rule('rule3', lambda x: x['now'], lt, sindicator)
    results = rule1.filter({"002597": {"now": 30, "low": 25,"high": 50}})
    results = rule2.filter(results)
    results = rule3.filter(results)
    print(results)