from setuptools import setup, find_packages
import sys
import os

version = '0.2.1'

long_description = open(
    os.path.join(os.path.dirname(__file__), 'README.rst')).read()

setup(name='morpfw',
      version=version,
      description="Web framework based on morepath",
      long_description=long_description,
      classifiers=[],  # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
      keywords='',
      author='Izhar Firdaus',
      author_email='izhar@kagesenshi.org',
      url='http://github.com/morpframework/morpfw',
      license='MIT',
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          # -*- Extra requirements: -*-
          'rulez>=0.1.2,<0.2.0',
          'more.cors',
          'celery',
          'redis',
          'jsl',
          'pyyaml>=4.2b1',
          'more.jsonschema',
          'jsonobject==0.9.4',
          'sqlalchemy',
          'more.signals',
          'DateTime',
          'transitions',
          'jsonpath_ng',
          'python-dateutil',
          'more.jwtauth',
          'more.itsdangerous',
          'sqlsoup',
          'celery',
          'gunicorn',
          'itsdangerous',
          'pyyaml',
          'passlib',
          'jsonschema',
          'more.transaction',
          'zope.sqlalchemy',
          'python-dateutil',
          'more.cors',
          'sqlalchemy_jsonfield',
          'sqlsoup',
          'celery',
          'gunicorn',
          'itsdangerous',
          'pyyaml',
          'passlib',
          'jsonschema',
          'more.transaction',
          'zope.sqlalchemy',
          'more.basicauth',
          'cryptography',
          'elasticsearch>=5.0.0,<6.0.0',
          'pamela',
          'click',
          'cookiecutter'
      ],
      extras_require={
          'test': [
              'nose',
              'webtest',
              'pytest==3.10.1',
              'pytest-html',
              'pytest_postgresql',
              'pytest_rabbitmq',
              'pytest-annotate',
              'pytest-cov',
              'pika',
          ]
      },
      entry_points={
          'morepath': ['scan=morpfw'],
          'console_scripts': [
              'morpfw=morpfw.cli:cli',
          ]
      }
      )


# -*- coding: utf-8 -*-
"""
Created on Wed Feb  3 00:00:55 2016

@author: Aastha
"""
import collections
import pandas as pd
from nltk import word_tokenize
from nltk.stem import SnowballStemmer
from sklearn.cluster import KMeans
from sklearn.feature_extraction.text import TfidfVectorizer
import xlsxwriter
from datetime import datetime

def process_text(text,stem=True):
    """ Tokenize text and stem words removing punctuation """
    text = text.translate("None")
    tokens = word_tokenize(text)
    if stem:
        stemmer = SnowballStemmer("english")
        tokens = [stemmer.stem(t) for t in tokens]

    return tokens

def cluster_texts(texts,clusters=3):
    """ Transform texts to Tf-Idf coordinates and cluster texts using K-Means """
    stopwords_custom={'the', 'these', 'ourselves', 'hers', 'are', 'most', 'by', 'other', 'i', 'so', 'having', 'our', 'under', 'did', 'whom', 'will',
                  'such', 'don', 'do', 'can', 't', 'where', 'once', 'during', 'its', 'to', 'how', 'this', 'they', 'against', 'himself', 'for', 'all', 'yourselves',
                  'while', 'but', 'that', 'any', 'had', 'again', 'more', 'very', 'am', 'it', 'before', 'my', 'we', 'herself', 'yours', 'as','there','here', 'only',
                  'because', 'into', 'than', 'have', 'from', 'of', 'an', 'above', 'has', 'your', 'now', 'which', 'theirs', 'you', 'and', 'few', 'ours', 'with', 's',
                  'up', 'off', 'yourself', 'on', 'their', 'at', 'after', 'out', 'themselves', 'if', 'why', 'was', 'not', 'through', 'then', 'in', 'were', 'itself',
                  'what', 'does', 'a', 'between', 'when', 'same', 'over', 'been', 'who', 'just', 'down', 'each', 'too', 'own', 'is', 'be', 'doing', 'below',
                  'both', 'myself', 'further', 'those', 'should', 'about', 'until', 'nor', 'some', 'being','he','also','she','would','.',',','...','like',';'}
#       tokens2 = [w for w in tokens1 if not w in stopwords_custom] # remove stop words

    print(datetime.strftime(datetime.now(), '%H:%M:%S')+"Clustering Begins")
    vectorizer = TfidfVectorizer(tokenizer=process_text,max_features=200000,
                                 stop_words=stopwords_custom,
                                 max_df=0.6,
                                 min_df=0.01
                                 )

    tfidf_model = vectorizer.fit_transform(texts)
    km_model = KMeans(n_clusters=clusters)
    km_model.fit(tfidf_model)
# to get top ten terms per cluster
    order_centroids = km_model.cluster_centers_.argsort()[:, ::-1]
    terms = vectorizer.get_feature_names()
    for i in range(0,clusters):
        print(i)
        for ind in order_centroids[i, :5]:
            print(terms[ind]," ,",end='')
            print()

# to write into file
    labels=km_model.predict(tfidf_model)
    clusters = {}
    n = 0

    book = xlsxwriter.Workbook('clust_sent.xlsx')
    sheet = book.add_worksheet()

    for item in labels:
        if item in clusters:
            clusters[item].append(texts[n])
        else:
            clusters[item] = [texts[n]]
        n +=1
    print(datetime.strftime(datetime.now(), '%H:%M:%S')+"Writing to File")
    j=0
    for item in clusters:
        print (item ," ,",end='')
        for i in clusters[item]:
            sheet.write(j, 0, int(item))
            sheet.write(j, 1, i)
            j=j+1

    book.close()
# To return clustering function
    clustering = collections.defaultdict(list)
    for idx, label  in enumerate(km_model.labels_):
        clustering[label].append(idx)

    return clustering

if __name__ == "__main__":
    print(datetime.strftime(datetime.now(), '%H:%M:%S')+"Reading File")
    file=pd.read_excel("C:\\Users\\Aastha\\Desktop\\sentiment.xlsx")
    articles= file.Description
    print(datetime.strftime(datetime.now(), '%H:%M:%S')+"File Read")
    clusters = cluster_texts(articles,6)
#    print(dict(clusters))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""The setup script."""
import warnings
from setuptools import setup, find_packages, Command
from setuptools.command.install import install as _install

with open('README.md') as readme_file:
    readme = readme_file.read()

requirements = [
    'requests',
    'Click'
]

setup_requirements = [
    'pytest-runner'
]

test_requirements = [
    'pytest'
]

class install(_install):
    def run(self):
        # _install.run(self)
        _install.do_egg_install(self)

class FakeBdist(Command):
    """Fake bdist wheel class for ignoring bdist_wheel build
    """

    user_options = [(
        "dist-dir=",
        "d",
        "directory to put final built distributions in [default: dist]"
    ), (
        "python-tag=",
        None,
        "Python tag (cp32|cp33|cpNN) for abi3 wheel tag (default:false)"
    )]

    def initialize_options(self):
        self.dist_dir = None
        self.python_tag = None

    def finalize_options(self):
        pass

    def run(self):
        warnings.warn(
            "{name}{version} does not support building wheels".format(
                name=about["__title__"],
                version=about["__version__"]
            )
        )

setup(
    name='split-downloader',
    version='0.2.9',
    description="Parallel downloader",
    long_description=readme,
    author="Joe Paul",
    author_email='joeirimpan@gmail.com',
    url='https://github.com/joeirimpan/pyparallel',
    packages=find_packages(include=['pyparallel']),
    include_package_data=True,
    install_requires=requirements,
    entry_points='''
        [console_scripts]
        download=pyparallel:cli
    ''',
    license="MIT license",
    zip_safe=False,
    keywords='pyparallel',
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        "Programming Language :: Python :: 2",
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],
    test_suite='tests',
    tests_require=test_requirements,
    long_description_content_type='text/markdown',
    setup_requires=setup_requirements,
    cmdclass={"install": install, "bdist_wheel": FakeBdist}
)


# -*- coding: utf-8 -*-

'''
Author - Mohd Aquib
'''
import torch
import torch.nn as nn
import torchvision
import torchvision.models as models
import torchvision.transforms as transforms
import torch.optim as optim
from torchvision.utils import save_image
from PIL import Image
import matplotlib.pyplot as plt

model = models.vgg19(pretrained=True).features

device = 'cuda' if torch.cuda.is_available() else 'cpu'

class VGG(nn.Module):
    def __init__(self):
        super(VGG,self).__init__()
        self.req_features = ['0','5','10','19','28']
        self.model = models.vgg19(pretrained=True).features[:29]
    def forward(self,x):
        features = []
        for layer_num,layer in enumerate(self.model):
            x = layer(x)
            if(str(layer_num) in self.req_features):
                features.append(x)
        return features

def image_loader(path):
    image = Image.open(path)
    loader = transforms.Compose([transforms.Resize((512,512)), transforms.ToTensor()])
    image = loader(image).unsqueeze(0)
    return image.to(device,torch.float)

original_image = image_loader('/content/Nikhil.JPG')
style_image = image_loader('/content/style.jpg')

generated_image = original_image.clone().requires_grad_(True)

def calc_content_loss(gen_feat,orig_feat):
    content_l = torch.mean((gen_feat - orig_feat)**2)
    return content_l

def calc_style_loss(gen,style):
    batch_size,channel,height,width = gen.shape
    G = torch.mm(gen.view(channel,height*width),gen.view(channel,height*width).t())
    A = torch.mm(style.view(channel,height*width),style.view(channel,height*width).t())
    style_l = torch.mean((G-A)**2)
    return style_l

def calculate_loss(gen_features,orig_features,style_features):
    style_loss=content_loss=0
    for gen,con,style in zip(gen_features,orig_features,style_features):
        content_loss += calc_content_loss(gen,con)
        style_loss += calc_style_loss(gen,style)
    total_loss = alpha*content_loss + beta*style_loss
    return total_loss

model = VGG().to(device).eval()

epoch = 7000
lr = 0.004
alpha = 8
beta = 70
optimizer = optim.Adam([generated_image],lr=lr)

for i in range(epoch):
    gen_features = model(generated_image)
    orig_features = model(original_image)
    style_features = model(style_image)

    total_loss = calculate_loss(gen_features,orig_features,style_features)
    optimizer.zero_grad()
    total_loss.backward()
    optimizer.step()

    if(not(i%100)):
        print(total_loss)
        save_image(generated_image,'gen.png')


from collections import defaultdict
import functools
import json
import logging
# import newrelic.agent
import os
# from otxb_core_utils.config import ConfigUtils
import traceback
import werkzeug.exceptions

from . import utils

app_registry = {}
logger = logging.getLogger(__name__)
# SECRETO = ConfigUtils.get_value("auth-secret")

from flask import Response, request

def build_absolute_url(path, params):
    return ""

# for compatibility with Django
class HttpResponse(Response):
    def __init__(self, content, status=200, content_type='text/html', headers=None):
        super(HttpResponse, self).__init__(
            response=content,
            content_type=content_type,
            status=status,
            headers=headers or {},
        )

class FileResponse(Response):
    def __init__(self, response_object=None, content=None, content_type=None):
        if response_object:
            super(FileResponse, self).__init__(
                response=response_object.response,
                content_type=response_object.content_type,
                status=response_object.status,
            )
        else:
            super(FileResponse, self).__init__(
                response=content, content_type=content_type
            )

class RequestFile(object):
    def __init__(self, fobj):
        self.fobj = fobj

    @property
    def name(self):
        return self.fobj.filename

    @property
    def size(self):
        return -1

    def chunks(self):
        while True:
            data = self.fobj.read(1024*32)
            if data:
                yield data
            else:
                return

    def read(self, *args, **kwargs):
        return self.fobj.read(*args, **kwargs)

    def tell(self, *args, **kwargs):
        return self.fobj.tell(*args, **kwargs)

    def seek(self, *args, **kwargs):
        return self.fobj.seek(*args, **kwargs)

def get_file(_request, key):
    if key not in _request.files:
        return None

    return RequestFile(_request.files[key])

class FlaskUser(object):
    def __init__(self, username, user_id):
        self.username = username
        self.user_id = user_id
        self.id = user_id
        self.is_staff = False

    def is_authenticated(self):
        return self.id != 0

class JSONResponse(Response):
    # @newrelic.agent.function_trace()
    def __init__(self, data=None, detail=None, err=False, status=None, indent=None):
        if not status:
            status = 400 if err else 200

        self._data = data if data is not None else {}  # fixme causes error
        if detail:
            self._data['detail'] = detail

        super(JSONResponse, self).__init__(
            response=json.dumps(self._data, cls=utils.OurJSONEncoder, indent=indent),
            status=status,
            mimetype='application/json'
        )

class XMLResponse(Response):
    def __init__(self, data=None, detail=None, err=False, status=None, content=None):
        status = status or 200
        self.content = content if content is not None else ""

        super(XMLResponse, self).__init__(
            response=self.content,
            status=status,
            mimetype='application/xml'
        )

def default_login_method(request=None, **kwargs):
    if request.user is None:
        return FlaskUser('Anonymous', 0)
    else:
        return request.user
    # token = request.headers.get('authorization')
    # if 'authorization' in request.headers:
    #     from otxb_core_utils.api import auth0
    #     payload = auth0.token_payload(token)
    #     return FlaskUser(payload['username'], payload['user_id'])
    #
    # return

def app_proxy(sa, fn, fnname, config, urlroot, cleanup_callback=None):
    _require_login = config.get('require_login', False)
    _require_admin = config.get('require_admin', False)
    app_blob = {
        '_newrelic_group': os.path.join('/', urlroot).rstrip('/'),
        '_fnname': fnname,
       '_config': config,
    }
    app_blob['_args'], app_blob['_kwargs'], app_blob['_va'], app_blob['_kw'] = config['fn_args']
    app_blob['combined_args'] = list(app_blob['_kwargs']) + [(x, None) for x in app_blob['_args']]

    @functools.wraps(fn)
    def appfn(*fn_args, **fn_kwargs):
        doer = sa()

        try:
            if request.data:
                api_data = request.get_json()
            else:
                api_data = request.form
        except werkzeug.exceptions.BadRequest as e:
            logger.warn("werkzeug.exceptions.BadRequest: url=%r, data=%r, headers=%r", request.path, request.data, [])
            raise e

        url_data = request.args
        login_fn = _require_login if callable(_require_login) else default_login_method
        blob = {
            'fn_args': fn_args,
            'fn_kwargs': fn_kwargs,
            'path': request.path,
            'request': request,
            'api_data': api_data,
            'url_data': url_data,
            'user': login_fn(request=request, api_data=api_data, url_data=url_data)
        }

        # ensure user is logged in if required
        if (_require_login or _require_admin):
            if not blob['user']:
                return JSONResponse(detail='Authentication required', status=403)

            if callable(blob['user'].is_authenticated):
                ia = blob['user'].is_authenticated()
            else:
                ia = blob['user'].is_authenticated

            if not ia:
                return JSONResponse(detail='Authentication required', status=403)

        # ensure user is admin if required
        if _require_admin and not blob['user'].is_admin:
            return JSONResponse(detail="Authentication required", status=403)

        # This does not handle file uploads, fix later
        val = utils.process_api(fn, doer, app_blob, blob)
        if cleanup_callback:
            try:
                cleanup_callback()
            except Exception as e:
                logger.error("Error performing cleanup callback: %r", traceback.format_exc())

        return val

    return appfn

def app_class_proxy(base_app, urlprefix, urlroot, app, cleanup_callback=None):
    counts = defaultdict(int)
    app_registry[urlroot] = app
    for fnname in app._api_functions():
        for version in app._api_versions():
            fn = app._fn(fnname, version)
            config = app._get_config(fn)

            for u in utils.urls_from_config(urlroot, fnname, fn, config, canonical=True):
                u = os.path.join("/", urlprefix, u.lstrip("/^").rstrip("/?$"))
                view_name = '{}-{}-{}-{}'.format(
                    urlprefix.replace('/', '_'),
                    urlroot.replace('/', '_'),
                    fnname,
                    counts[fnname]
                )
                logger.info("Adding url %r to %r", u, view_name)
                base_app.add_url_rule(
                    u,
                    endpoint=view_name,
                    view_func=app_proxy(app.__class__, fn, fnname, config, urlroot, cleanup_callback=cleanup_callback),
                    methods=['GET', 'POST', 'PATCH', 'PUT', 'DELETE', 'HEAD']
                )
                counts[fnname] += 1


import datetime
import os

import cv2
import shutil

import voc
import numpy as np

bath_path = r"\\192.168.55.38\Team-CV\cam2pick\camera_pic_0717\bj_test02"
pic_path = bath_path + "/rec_pic/"

xml_path = bath_path + "/Annotations/"

import numpy as np

xmls = os.listdir(xml_path)

samples=[]
repeat_count=[]
for path in xmls:
    boxs = voc.get_boxs(xml_path + path)

    time1=datetime.datetime.now()
    data = np.array(boxs)
    idex = np.lexsort([data[:, 3], data[:, 2], data[:, 1], data[:, 0]])
    sorted_data = data[idex, :]

    sorted_data=sorted_data.reshape(len(sorted_data) * 4)
    min_distence = 9999
    min_index=0
    if len(samples)>0:
        for inxdex, sample in enumerate(samples):
            a = sample
            b = sorted_data
            if len(sample) > len(sorted_data):
                b = np.zeros(len(sample), dtype=np.int)
                b[0:len(sorted_data)] = sorted_data
            elif len(sample) < len(sorted_data):
                a = np.zeros(len(sorted_data), dtype=np.int)
                a[0:len(sample)] = sample
            distance = np.linalg.norm(a - b)
            if (min_distence>distance):
                min_distence=distance
                min_index=inxdex
        if min_distence>3 and min_distence<9999:
            pass
            # print(pic_path + path[:-4] + ".jpg", min_distence)
            # shutil.copyfile(xml_path + path, "output/Annotations/"+path)
            # shutil.copyfile(pic_path + path[:-4] + ".jpg", "output/rec_pic/" + path[:-4] + ".jpg")
        elif repeat_count[min_index]<200:
            pass
            # print(pic_path + path[:-4] + ".jpg", repeat_count[min_index])
            # shutil.copyfile(xml_path + path, "output/Annotations/" + path)
            # shutil.copyfile(pic_path + path[:-4] + ".jpg", "output/rec_pic/" + path[:-4] + ".jpg")
        else:
            repeat_count[min_index]+=1

    if len(samples) > 5:
        min_index = repeat_count.index(min(repeat_count))
        samples[min_index] = sorted_data
        pass
    else:
        samples.append(sorted_data)
        repeat_count.append(0)
    print("time1",(datetime.datetime.now()-time1).microseconds)

class BoxRepeat():
    repeat_count = []

    samples = []
    def box_repeat(self,frame, bbox_mouse,timestr):
        time1 = datetime.datetime.now()
        if len(bbox_mouse)==0:
            return
        data = np.array(bbox_mouse)
        idex = np.lexsort([data[:, 3], data[:, 2], data[:, 1], data[:, 0]])
        sorted_data = data[idex, :]

        sorted_data = sorted_data.reshape(len(sorted_data) * 4)
        min_distence = 9999
        min_index = 0

        if len(self.samples) == 0:
            self.samples.append(sorted_data)
            self.repeat_count.append(0)
        else:
           for inxdex, sample in enumerate(self.samples):
               a = sample
               b = sorted_data
               if len(sample) > len(sorted_data):
                   b = np.zeros(len(sample), dtype=np.int)
                   b[0:len(sorted_data)] = sorted_data
               elif len(sample) < len(sorted_data):
                   a = np.zeros(len(sorted_data), dtype=np.int)
                   a[0:len(sample)] = sample
               distance = np.linalg.norm(a - b)
               if (min_distence > distance):
                   min_distence = distance
                   min_index = inxdex
           if min_distence > 3 and min_distence < 9999:
               if len(self.samples) > 5:
                   if max(self.repeat_count) > 50 and min(self.repeat_count) < 5:
                       min_index = self.repeat_count.index(min(self.repeat_count))
                       self.samples[min_index] = sorted_data
                       self.repeat_count[min_index] = 0
               else :
                   self.samples.append(sorted_data)
                   self.repeat_count.append(0)
               cv2.imwrite(self.rec_path + '../repeat_no/' + timestr + '.jpg', frame)
               # print(pic_path + path[:-4] + ".jpg", min_distence)
               # shutil.copyfile(xml_path + path, "output/Annotations/"+path)
               # shutil.copyfile(pic_path + path[:-4] + ".jpg", "output/rec_pic/" + path[:-4] + ".jpg")
           elif self.repeat_count[min_index] < 200:
               cv2.imwrite(self.rec_path + '../repeat2/' + timestr + '.jpg', frame)
               self.repeat_count[min_index] += 1
               print("repeat_count",min_index, self.repeat_count[min_index])
               # shutil.copyfile(xml_path + path, "output/Annotations/" + path)
               # shutil.copyfile(pic_path + path[:-4] + ".jpg", "output/rec_pic/" + path[:-4] + ".jpg")
           else:
               cv2.imwrite(self.rec_path + '../repeat200/' + timestr + '.jpg', frame)
               self.repeat_count[min_index] += 1
        print("box_repeat time1", (datetime.datetime.now() - time1).microseconds)
if __name__ == '__main__':
    box_re= BoxRepeat()

    xmls = os.listdir(xml_path)

    for path in xmls:
        boxs = voc.get_boxs(xml_path + path)
        data = np.array(boxs)

        box_re.box_repeat()

        time1 = datetime.datetime.now()


import time
from turtle import Screen

import simpleaudio as sa
from stars import ImageLabel
import pickle
from audio import Audio
from barrier import Barrier
from bullet import Bullet, Bullets
from invaders import Invaders
from tank import Tank
from zaps import Zaps
from screentext import Title, PressSpace, Hide, BottomLine, Lives, GameOver, Score, HiScore, Level, Paused

# ------------------- Sounds --------------------------------------------------- #

laser = sa.WaveObject.from_wave_file("audio/laser-2.wav")
shot = sa.WaveObject.from_wave_file("audio/shoot.wav")
inv_kill = sa.WaveObject.from_wave_file("audio/invaderkilled.wav")
splode = sa.WaveObject.from_wave_file("audio/explosion.wav")
ufo_wav = sa.WaveObject.from_wave_file("audio/ufo_lowpitch.wav")

move_sound_i = 0

# --------------------------------- functions ------------------------------------------ #

def done():
    """
    Quits the game and writes the audio.time and audio.list to a .txt and.pkl
    file respectively so they can be accessed by video-main.py to create a video
    of the gameplay (if video-main.py is running).
    :return:
    """
    audio.running = False

    str_time = str(audio.time)
    with open("audiolist.pkl", "wb") as file:
        pickle.dump(audio.list, file)
    with open("audiotime.txt", "w")  as file:
        file.write(str_time)
    screen.bye()

def move_sound():
    """
    Every time the invzders move this function plays  one of four
    move sounds and cycles to the next sound in the list. Once all four sounds have
    been played it resets to the first sound.
    :return:
    """
    global move_sound_i
    move_sounds = [sa.WaveObject.from_wave_file("audio/1.wav"),
                   sa.WaveObject.from_wave_file("audio/2.wav"),
                   sa.WaveObject.from_wave_file("audio/3.wav"),
                   sa.WaveObject.from_wave_file("audio/4.wav"),
                   ]
    if invaders.moved:
        move_sounds[move_sound_i].play()
        ad = {"file": f"audio/{move_sound_i+1}.wav",
              "time": (time.time() - audio.time) }

        audio.list.append(ad)
        if move_sound_i == 3:
            move_sound_i = 0
        else:
            move_sound_i += 1

def move_handler(x):
    """
    allows the tank sprite to track the cursor using the Screen.onmove method
    I've added to the turtle library
    :param x: x axis value of the mouse cursor
    :return:
    """
    screen.onmove(None)  # avoid overlapping events
    tank.penup()
    if not invaders.pause:
        if 330 > x > -330:
            tank.setheading(tank.towards(x, -250))
            tank.goto(x, -275)
        screen.onmove(move_handler)
    else:
        pass

def shoot(x, y):
    """
    creates a Bullet object at time and position of click and plays the shot wav file.
    :param x:
    x axis position of cursor at time of click
    :param y:
    y axis position of cursor at time of click
    :return:
    """
    bullets.b_list.append(Bullet(x))
    shot.play()
    ad = {"file": f"audio/shoot.wav",
          "time": (time.time() - audio.time)}
    audio.list.append(ad)

def pause():
    """
    Tied to screen.onkey("p").
    Stops all inputs and motion and displays "PAUSED"  when the "p" key is pressed while
    preserving game state.  resumes when unpaused.
    :return:
    """
    if not invaders.pause:
        invaders.pause = True
        zaps.pause = True
        paused.showturtle()
        for z in zaps.z_list:
            z.pause = True
        for b in bullets.b_list:
            b.pause = True
        screen.onclick(None)
    else:
        invaders.pause = False
        zaps.pause = False
        paused.hideturtle()
        for z in zaps.z_list:
            z.pause = False
        for b in bullets.b_list:
            b.pause = False
        screen.onclick(shoot)

def bullet_handler():
    """
    Handles the Bullet class objects in the Bullets object b_list.
    detects collisions with Invader objects and Brick objects in the barriers, as well
    as when the bullet reaches the upper bounds of the screen.
    :return:
    """
    if len(bullets.b_list) >= 1:  # limit number of bullets on screen
        screen.onclick(None)
    else:
        if not invaders.pause:
            screen.onclick(shoot)
    for b in bullets.b_list:
        b.move()
        if b.distance(invaders.saucer) < 30 and b.ycor() > (invaders.saucer.ycor() + 2):
            score.count += invaders.saucer.points
            score.draw()
            invaders.saucer.current_shape = invaders.saucer.shape_2
            invaders.saucer.shape(invaders.saucer.current_shape)
            inv_kill.play()
            ad = {"file": "audio/invaderkilled.wav",
                  "time": (time.time() - audio.time)}
            audio.list.append(ad)
            b.hideturtle()
            try:
                bullets.b_list.remove(b)
            except ValueError:
                pass

        for br in barrier.brick_list:
            if br.isvisible():
                if b.distance(br) < 10 and b.ycor() < br.ycor() + 2:
                    b.hideturtle()
                    b.goto(5000, 5000)
                    if br.current_shape == br.shape_1:
                        br.current_shape = br.shape_2
                        br.shape(br.current_shape)
                    else:
                        br.hideturtle()
        for col in invaders.invader_list:
            for inv in col:
                if inv.isvisible():
                    if inv.current_shape == inv.shape_1 or inv.current_shape == inv.shape_2:
                        if b.distance(inv) < 20 and b.ycor() < (inv.ycor() + 2):
                            score.count += inv.points
                            score.draw()
                            inv.current_shape = inv.shape_3
                            inv.shape(inv.current_shape)
                            inv_kill.play()
                            ad = {"file": "audio/invaderkilled.wav",
                                  "time": (time.time() - audio.time)}
                            audio.list.append(ad)
                            b.hideturtle()
                            try:
                                b.hideturtle()
                                bullets.b_list.remove(b)
                            except ValueError:
                                pass
        if b.ycor() > 330:  # delete offscreen bullets
            b.shape("sprites/bullet-splode.gif")
            bullets.dead_b_list.append(b)
            try:
                bullets.b_list.remove(b)
            except ValueError:
                pass
    for b in bullets.dead_b_list:
        if b.timer == 0:
            b.hideturtle()
            bullets.dead_b_list.remove(b)
        else:
            b.timer -= 1

def game_over():
    """
    Displays the game over text and returns the game to it's initial level-01 state.
    :return:
    """
    invaders.timer_abs = 22
    level.number = 1
    invaders.interval = 1.7
    invaders.init_y = 270
    invaders.saucer_count = 10
    zaps.limiter_abs = 20
    invaders.saucer.hideturtle()
    invaders.saucer.goto(-380, 330)
    invaders.saucer.showturtle()
    invaders.saucer_count = 0
    for b in bullets.b_list:
        b.hideturtle()
        b.goto(1000, 1000)
    for zap in zaps.z_list:
        zap.hideturtle()
        zap.goto(1000, 1000)
    zaps.z_list = []
    hide.showturtle()
    g_over.showturtle()
    hi_score.save_hi_score(score.count)
    hi_score.draw()
    screen.update()
    barrier.show()
    invaders.reset()
    screen.onkey(fun=game, key="space")
    hide.showturtle()
    g_over.showturtle()
    tank.shape("sprites/tank-blue.gif")
    screen.update()
    score.count = 0
    lives.count = 3
    print(audio.list)

def next_level():
    """
    When every Invader object has been destroyed (i.e. made not visible with hideturtle())
    resets the invaders, barriers, bottom boundary line, and changes variables
    (invader move speed, invader starting position, invader shot speed, etc.)
    to increase the difficulty for the next level.
    :return:
    """

    hide.showturtle()
    screen.update()
    screen.onclick(None)
    level.number += 1
    invaders.interval -= .1
    invaders.timer_abs = 22 - level.number
    invaders.init_y -= 35
    invaders.saucer_count += 1
    zaps.limiter_abs -= 2
    for b in bullets.b_list:
        b.hideturtle()
        b.goto(1000, 1000)
    for zap in zaps.z_list:
        zap.hideturtle()
        zap.goto(1000, 1000)
    zaps.z_list = []
    hide.showturtle()
    level.draw()
    invaders.reset()
    barrier.show()
    time.sleep(2)

def game():
    '''
    starts and runs the standard game mode.  loops while the invaders.game
    attribute is true.
    :return:
    invaders.game = False
    '''
    gif_window.load("stars.gif")
    press_space.title_screen = False
    invaders.game = True
    BottomLine()
    screen.onkey(fun=None, key="space")
    score.draw()
    lives.draw()
    level.clear()
    g_over.hideturtle()
    title.hideturtle()
    press_space.hideturtle()
    hide.hideturtle()
    invaders.start_time = time.time()
    screen.onkey(fun=done, key="q")
    screen.onkey(fun=pause, key="p")
    screen.onclick(shoot)
    screen.listen()
    screen.update()
    while invaders.game:
        gif_window.next_frame(gif_window.loc, gif_window.frame_no)
        invaders.completed = True
        screen.onmove(move_handler)
        screen.tracer(0)
        invaders.move()
        bullet_handler()
        move_sound()
        zaps.timer()
        # play sound when UFO comes on screen
        if invaders.saucer.xcor() == -371:
            ufo_wav.play()
            ad = {"file": f"audio/ufo_lowpitch.wav",
                  "time": (time.time() - audio.time)}
            audio.list.append(ad)
        # Handles the Invader objects in invaders.invader_list:
        #  - Checks if they are currently visible
        #  - Has them fire if the ank is underneath (and moving... a bug i haven't been able to fix)
        #  - Checks for collision with the barriers and (if you've really messed up) the tank.
        for col in invaders.invader_list:
            for i in range(0, 5):
                inv = col[i]
                if inv.isvisible():
                    invaders.completed = False
                    if abs(tank.xcor() - inv.xcor()) < 5:
                        try:
                            if not col[i+1].isvisible():
                                zaps.zap_em(x=inv.xcor(), y=inv.ycor())
                        except IndexError:
                            zaps.zap_em(x=inv.xcor(), y=inv.ycor())
                    for br in barrier.brick_list:
                        if br.isvisible():
                            if inv.distance(br) < 17 and inv.ycor() > (br.ycor() - 2):
                                br.hideturtle()

                    if inv.distance(tank) < 17 and inv.ycor() > (tank.ycor() - 2):
                        lives.count = 0
                        lives.draw()
                        screen.update()
                        game_over()
                        invaders.game = False
                        return

        # Handles the Zap (what i called the Invaders shots) objects in zaps.zap_list:
        #  - Checks for collision with the barriers and the tank.
        #  - if the tank is hit destroys the tank, plays the explosion sound, lowers lives, etc
        #  - triggers game ver if lives reach 0
        #  - lastly check if the zap has hit the bottom line, changes zap into bottom line damage sprite.
        for z in zaps.z_list:
            z.move()
            if z.current_shape == z.shape_1:
                z.current_shape = z.shape_2
                z.shape(z.current_shape)
            else:
                z.current_shape = z.shape_1
                z.shape(z.current_shape)
            if z.distance(tank) < 20 and z.ycor() > (tank.ycor() - 2):
                screen.onmove(None)
                lives.count -= 1
                lives.draw()
                screen.update()
                splode.play()
                ad = {"file": f"audio/explosion.wav",
                      "time": (time.time() - audio.time)}
                audio.list.append(ad)
                z.hideturtle()
                z.goto(5000, 5000)
                try:
                    zaps.z_list.remove(z)
                except ValueError:
                    pass
                for i in range(5):
                    tank.shape("sprites/splode-1.gif")
                    screen.update()
                    time.sleep(.1)
                    tank.shape("sprites/splode-2.gif")
                    screen.update()
                    time.sleep(.1)
                if lives.count <= 0:
                    game_over()
                    invaders.game = False
                    return
                screen.onmove(move_handler)
                tank.shape("sprites/tank-blue.gif")
            for br in barrier.brick_list:
                if br.isvisible():
                    if z.distance(br) < 12 and z.ycor() > (br.ycor() - 2):
                        print(len(barrier.brick_list))
                        z.hideturtle()
                        z.goto(5000, 5000)
                        try:
                            zaps.z_list.remove(z)
                        except ValueError:
                            pass
                        if br.current_shape == br.shape_1:
                            br.current_shape = br.shape_2
                            br.shape(br.current_shape)
                        else:
                            br.hideturtle()
            if z.ycor() <= -310:
                z.goto(z.xcor(), -303)
                z.shape("sprites/ishot-dead.gif")
                try:
                    zaps.z_list.remove(z)
                except ValueError:
                    pass
        if invaders.completed:
            next_level()
            game()
        screen.update()
        time.sleep(0.0)

# ----------------------- Create Objects, setup --------------------#

screen = Screen()
screen.title("Space Invaders")
screen.setup(680, 800)
screen.bgcolor("black")
canvas = screen.getcanvas()
gif_window = ImageLabel(canvas)

lives = Lives()
tank = Tank()
bullets = Bullets()
zaps = Zaps()
invaders = Invaders()
barrier = Barrier()
score = Score()
hi_score = HiScore()
hide = Hide()
g_over = GameOver()
paused = Paused()
level = Level()

screen.listen()
screen.onkey(fun=game, key="space")
audio = Audio()
BottomLine()
title = Title()
press_space = PressSpace(screen)
press_space.flash()

screen.mainloop()


import os
import shutil
from datetime import datetime
from unittest import mock

import pytz
from django import forms
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import Client, TestCase, override_settings
from django.urls import reverse

from posts.apps import PostsConfig
from posts.models import Follow, Group, Post, User
from yatube.settings import BASE_DIR

temp_media = os.path.join(BASE_DIR, "temp_media")

@override_settings(MEDIA_ROOT="temp_media")
class PostPagesTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        # Создадим запись в БД:
        cls.user = User.objects.create(
            first_name="test_name",
            last_name="test_family_name",
            username="test_user",
            email="test_user@test_site.ru"
        )
        cls.user_2 = User.objects.create(
            username="test_user_2",
        )
        cls.user_3 = User.objects.create(
            username="test_user_3",
        )
        cls.group = Group.objects.create(
            title="Тест | Название",
            description="Тест | Описание группы",
            slug="test-slug"
        )
        cls.group_2 = Group.objects.create(
            title="Тест | Название 2",
            description="Тест | Описание группы 2",
            slug="test-slug-2"
        )

        small_gif = (
            b'\x47\x49\x46\x38\x39\x61\x02\x00'
            b'\x01\x00\x80\x00\x00\x00\x00\x00'
            b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
            b'\x00\x00\x00\x2C\x00\x00\x00\x00'
            b'\x02\x00\x01\x00\x00\x02\x02\x0C'
            b'\x0A\x00\x3B'
        )
        cls.uploaded_gif = SimpleUploadedFile(
            name="small.gif",
            content=small_gif,
            content_type="image/gif"
        )

        cls.post_on_page = PostsConfig.pages_on_list

        cls.mocked = datetime(2021, 1, 1, 0, 0, 0, tzinfo=pytz.utc)
        with mock.patch('django.utils.timezone.now',
                        mock.Mock(return_value=cls.mocked)):
            cls.post_1 = Post.objects.create(
                text="Тест | текст | " * 10,
                author=cls.user,
                group=cls.group,
                image=cls.uploaded_gif
            )
            for i in range(1, cls.post_on_page):
                Post.objects.create(
                    text=f"{i}",
                    author=cls.user,
                )
            cls.post_user_2 = Post.objects.create(
                text="Author",
                author=cls.user_2,
            )

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(temp_media, ignore_errors=True)
        super().tearDownClass()

    def setUp(self):
        self.guest_client = Client()
        self.user = PostPagesTests.user
        self.user_2 = PostPagesTests.user_2
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)

    def test_pages_uses_correct_template(self):
        """URL-адрес использует соответствующий шаблон."""
        post_pk = PostPagesTests.post_1.pk
        user = PostPagesTests.user
        group = PostPagesTests.group

        templates_pages_names = {
            reverse("index"): "index.html",
            reverse("new_post"): "new.html",
            reverse("group", kwargs={"slug": group.slug}): "group.html",
            reverse("post", kwargs={
                "post_id": post_pk,
                "username": user.username
            }): "post.html",
            reverse("profile", kwargs={"username": user}): "profile.html",
            reverse("post_edit", kwargs={
                "post_id": post_pk,
                "username": user.username
            }): "new.html"}

        for reverse_name, template in templates_pages_names.items():
            with self.subTest(reverse_name=reverse_name):
                response = self.authorized_client.get(reverse_name)
                self.assertTemplateUsed(response, template)

    def test_index_page_list_is_10(self):
        post_on_page = PostPagesTests.post_on_page
        response = self.authorized_client.get(reverse("index"))
        self.assertEquals(len(response.context["page"]), post_on_page)

    def test_home_page_show_correct_context(self):
        """Шаблон index сформирован с правильным контекстом."""
        mocked = PostPagesTests.mocked
        response = self.authorized_client.get(reverse("index"))
        user = PostPagesTests.user
        post = PostPagesTests.post_1
        group = PostPagesTests.group
        post_0 = response.context["page"][0]
        post_image = post_0.image
        test_dict = {
            post_0.text: post.text,
            post_0.author.get_full_name(): user.get_full_name(),
            post_0.pub_date: mocked,
            post_0.group.title: group.title,
            post_image: "posts/small.gif",
        }
        for value, expected in test_dict.items():
            with self.subTest(value=value):
                self.assertEquals(value, expected)

    def test_group_detail_pages_show_correct_context(self):
        """Шаблон group сформирован с правильным контекстом."""
        mocked = PostPagesTests.mocked
        user = PostPagesTests.user
        post = PostPagesTests.post_1
        group = PostPagesTests.group
        group_2 = PostPagesTests.group_2

        response = self.authorized_client.get(
            reverse("group", kwargs={"slug": "test-slug"})
        )
        self.assertEquals(response.context["group"].title, group.title)
        self.assertEquals(response.context["group"].description,
                          group.description)

        post_0 = response.context["page"][0]
        post_text_0 = post_0.text
        post_author_0 = post_0.author.get_full_name()
        post_pub_date_0 = post_0.pub_date
        post_group_0 = post_0.group.title
        post_image = post_0.image
        self.assertEquals(post_text_0, post.text)
        self.assertEquals(post_author_0, user.get_full_name())
        self.assertEquals(post_pub_date_0, mocked)
        self.assertEquals(post_group_0, group.title)
        self.assertNotEqual(post_group_0, group_2.title)
        self.assertEquals(post_image, "posts/small.gif")

    def test_new_page_show_correct_context(self):
        """Шаблон new сформирован с правильным контекстом."""
        response = self.authorized_client.get(reverse("new_post"))
        form_fields = {
            "group": forms.fields.ChoiceField,
            "text": forms.fields.CharField,
        }

        for value, expected in form_fields.items():
            with self.subTest(value=value):
                form_field = response.context["form"].fields.get(value)
                self.assertIsInstance(form_field, expected)

    def test_username_post_id_edit_context(self):
        """Шаблон post_id_edit сформирован с правильным контекстом."""
        post_pk = PostPagesTests.post_1.pk
        response = self.authorized_client.get(
            reverse("post_edit",
                    kwargs={"post_id": post_pk,
                            "username": self.user.username})
        )
        form_fields = {
            "group": forms.fields.ChoiceField,
            "text": forms.fields.CharField,
        }
        for value, expected in form_fields.items():
            with self.subTest(value=value):
                form_field = response.context["form"].fields.get(value)
                self.assertIsInstance(form_field, expected)

    def test_profile_pages_show_correct_context(self):
        """Шаблон profile сформирован с правильным контекстом."""
        mocked = PostPagesTests.mocked
        user = PostPagesTests.user
        post = PostPagesTests.post_1
        group = PostPagesTests.group

        response = self.authorized_client.get(reverse("profile", kwargs={
            "username": self.user.username,
        }))

        post_0 = response.context["page"][0]
        post_text_0 = post_0.text
        post_author_0 = post_0.author.get_full_name()
        post_pub_date_0 = post_0.pub_date
        post_group_0 = post_0.group.title
        post_image = post_0.image
        self.assertEquals(post_text_0, post.text)
        self.assertEquals(post_author_0, user.get_full_name())
        self.assertEquals(post_pub_date_0, mocked)
        self.assertEquals(post_group_0, group.title)
        self.assertEquals(post_image, "posts/small.gif")

    def test_post_id_show_correct_context(self):
        """Шаблон post_id сформирован с правильным контекстом."""
        mocked = PostPagesTests.mocked
        user = PostPagesTests.user
        post = PostPagesTests.post_1
        group = PostPagesTests.group

        response = self.authorized_client.get(reverse("post", kwargs={
            "username": self.user.username,
            "post_id": post.pk,
        }))
        response = response.context["post"]
        post_text_0 = response.text
        post_author_0 = response.author.get_full_name()
        post_pub_date_0 = response.pub_date
        post_group_0 = response.group.title
        post_image = response.image
        self.assertEquals(post_text_0, post.text)
        self.assertEquals(post_author_0, user.get_full_name())
        self.assertEquals(post_pub_date_0, mocked)
        self.assertEquals(post_group_0, group.title)
        self.assertEquals(post_image, "posts/small.gif")

    def test_first_page_contains_ten_records(self):
        """Проверка: количество постов на первой странице равно 10."""
        response = self.client.get(reverse("index"))
        pages = PostPagesTests.post_on_page
        self.assertEquals(len(response.context["page"].object_list), pages)

    def test_auth_user_follow(self):
        user = PostPagesTests.user
        user_2 = PostPagesTests.user_2

        self.authorized_client.get(
            reverse("profile_follow", kwargs={"username": user_2.username})
        )
        self.assertEquals(Follow.objects.get(user=user).author, user_2)

    def test_auth_user_unfollow(self):
        user = PostPagesTests.user
        user_2 = PostPagesTests.user_2

        self.authorized_client.get(
            reverse("profile_follow", kwargs={"username": user_2.username})
        )
        self.authorized_client.get(
            reverse("profile_unfollow", kwargs={"username": user_2.username})
        )
        self.assertFalse(Follow.objects.filter(user=user).exists())

    def test_new_post_in_following(self):
        user_2 = PostPagesTests.user_2
        user_3 = PostPagesTests.user_3
        post = PostPagesTests.post_user_2

        self.authorized_client.get(
            reverse("profile_follow", kwargs={"username": user_2.username})
        )
        response = self.authorized_client.get(reverse("follow_index"))
        post_0 = response.context["page"][0]
        post_text_0 = post_0.text
        self.assertEquals(post_text_0, post.text)

        self.authorized_client.force_login(user_3)
        response = self.authorized_client.get(reverse("follow_index"))
        self.assertEquals(len(response.context["page"]), 0)


import os
import random
from sh import git, ghp_import

import pyladies_cz

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
OUTPUT_DIR = os.path.join(ROOT_DIR, '_build')

COMMIT_EMOJIS = [
    ':sunglasses:', ':two_hearts:', ':sparkles:', ':star2:', ':star:',
    ':blue_heart: :yellow_heart:', ':raised_hands:', ':ok_hand:', ':ok_woman:',
    ':dancer:', ':raising_hand:', ':woman:', ':girl:', ':princess:', ':sunny:',
    ':cat:', ':koala:', ':snake:', ':paw_prints:', ':four_leaf_clover:',
    ':octocat:', ':gift_heart:', ':tada:', ':balloon:', ':cake:', ':rocket:',
]

def deploy():
    print('Generating HTML...')
    pyladies_cz.freeze_app(path=OUTPUT_DIR, base_url='http://pyladies.cz')

    if os.environ.get('TRAVIS'):  # Travis CI
        print('Setting up Git...')
        git.config(
            'user.name',
            git('show', format='%cN', s=True, _tty_out=False).strip()
        )
        git.config(
            'user.email',
            git('show', format='%cE', s=True, _tty_out=False).strip()
        )

        github_token = os.environ.get('GITHUB_TOKEN')  # encrypted in .travis.yml
        repo_slug = os.environ.get('TRAVIS_REPO_SLUG')
        origin = 'https://{}@github.com/{}.git'.format(github_token, repo_slug)
        git.remote('set-url', 'origin', origin)

    print('Rewriting gh-pages branch...')
    commit_message = 'Deploying {}'.format(random.choice(COMMIT_EMOJIS))
    ghp_import('-n', '-m', commit_message, OUTPUT_DIR)  # -n includes .nojekyll file

    print('Pushing to GitHub...')
    git.push('origin', 'gh-pages:gh-pages', force=True)

if __name__ == '__main__':
    deploy()


import pandas as pd
import numpy as np
import os
from .visualize import feature_plot, eva_plot
from . import feature_evaluation
from .feature_explore import *
import simplejson
from itertools import chain
from collections import defaultdict
from ..monitor import data_monitor as dm
import pickle
import traceback
from ..base_utils import *
# from ...service.base_utils import *
from ..data_service import get_data
from ..monitor.psi import cmpt_psi
from .feature_explore import DescStats

# TODO: report各函数df_test应该作为可选参数
# TODO: 计算psi时目前只考虑了两个数据集、其中之一为base的情况，对于多数据集情况没有考虑和适应。

def _update_dict(s, k1, k2):
    """
    json to dict，dict值转成频率，外部套上对应数据集名称
    :param s:
    :param k1:
    :param k2:
    :return:
    """
    dic = json.loads(s[k2])
    total = sum(dic.values())
    dic = {k: v / total for k, v in dic.items()}
    return {s[k1]: dic}

def _merge_dict(s):
    """
    不同字典key值对齐，不存在的key填0
    :param s:
    :return:
    """
    dic = {}
    keys_list = []
    for i in s:
        for k, dic_i in i.items():
            keys_list += list(dic_i.keys())
    keys_list = set(keys_list)

    for i in s:
        for outk, dic_i in i.items():
            ii = {k: dic_i.get(k, 0) for k in keys_list}
            i[outk] = ii
        dic.update(i)
    return json.dumps(dic)

def _get_single_plot_data(s, k1, k2):
    return json.dumps(json.loads(s[k2])[s[k1]])

def _get_distribution_psi(s):
    return cmpt_psi(list(json.loads(s[0]).values()), list(json.loads(s[1]).values()))

def _gen_plot_data(df, k, set_index=False):
    tmp = df.reset_index()

    tmp[k] = tmp.apply(lambda s: _update_dict(s, df.index.names[-1], k), axis=1)
    tmp2 = tmp.groupby(df.index.names[0])[k].agg(_merge_dict)
    tmp2 = pd.DataFrame(tmp2)
    tmp2.columns = ['plot']
    tmp = df.join(tmp2, how='outer')
    tmp = tmp.reset_index()
    tmp['distribution'] = tmp.apply(lambda s: _get_single_plot_data(s, df.index.names[-1], 'plot'), axis=1)
    tmp = tmp.set_index(df.index.names)

    t = tmp.groupby(level=0)['distribution'].agg(_get_distribution_psi)
    t.name = 'H-PSI'
    tmp = tmp.join(t, how='outer')
    col_list = list(tmp.columns)
    col_list = col_list[-1:] + col_list[:-3] + col_list[-3:-1]
    tmp = tmp[col_list]

    if set_index:
        tmp = tmp.set_index('plot', append=True).swaplevel(-2, -1)
    return tmp

def _gen_desc_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=False):
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()
    # data_stats
    enc = DescStats()
    enc.fit(df_train)
    c_tr = enc.transform(df_train)
    c_te = enc.transform(df_test)
    df = merge_multi_df({'train': c_tr, 'test': c_te}, axis=0).round(decimals)
    df = _gen_plot_data(df, 'distribution')
    # TODO: plot列应该放到索引中以图片形式呈现，因为暂时没有解决excel画图插入cell问题，先删除
    del df['plot']

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'data_stats.report'))
        dic = {'train': table_to_dict(c_tr), 'test': table_to_dict(c_te)}
        simplejson.dump(dic, open(os.path.join(report_dst, 'data_stats.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
    return df

def _gen_woe_report(df_train_origin, df_test_origin, label_col, report_dst, decimals=4, gen_report=False):
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()
    # woe_report
    wr = feature_evaluation.WOEReport()
    d_tr = wr.fit_transform(df_train, label_col)
    d_te = wr.transform(df_test, label_col)
    df = merge_multi_df({'train': d_tr, 'test': d_te}, axis=1)

    df['group_sample_ratio'] = df['group_sample_ratio'].fillna(0).astype(float)
    df['total_sample_num'] = df['total_sample_num'].fillna(0).astype(float)
    df['overdue_sample_num'] = df['overdue_sample_num'].fillna(0).astype(float)
    df['overdue_ratio'] = df['overdue_ratio'].fillna(0).astype(float)

    t = df.iloc[:, [i for i, v in enumerate(list(df.columns.labels[0])) if v == list(df.columns.levels[0]).index('IV')]]
    del df['IV']
    df = t.join(df, how='outer')
    df.insert(0, 'L-PSI', value=np.nan)
    df.insert(0, 'V-PSI', value=np.nan)
    for i in df.index.levels[0]:
        # TODO： 以下写法在train和test数据集index有重复时会报错,why?
        # df.loc[i, 'V-PSI'] = cmpt_psi(df.loc[i, 'group_sample_ratio']['train'],
        #                               df.loc[i, 'group_sample_ratio']['test'])
        # df.loc[i, 'L-PSI'] = cmpt_psi(df.loc[i, 'overdue_ratio']['train'],
        #                               df.loc[i, 'overdue_ratio']['test'])
        df.loc[i, 'V-PSI'] = cmpt_psi(df.loc[i]['group_sample_ratio']['train'],
                                      df.loc[i]['group_sample_ratio']['test'])
        df.loc[i, 'L-PSI'] = cmpt_psi(df.loc[i]['overdue_ratio']['train'],
                                      df.loc[i]['overdue_ratio']['test'])
    df.round(decimals)

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'woe_stats.report'))
    return df

def _gen_eva_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=False):
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()
    # feature_indice
    train_data = feature_evaluation.feature_evaluation(df_train, [label])
    test_data = feature_evaluation.feature_evaluation(df_test, [label])

    df = merge_multi_df({'train': train_data, 'test': test_data}, axis=1).round(decimals=decimals)

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'feature_indices.report'))

        text = {'train': table_to_dict(train_data), 'test': table_to_dict(test_data)}
        simplejson.dump(text, open(os.path.join(report_dst, 'feature_indices.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
    return df

def _gen_woe_eva_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=True):
    df1 = _gen_woe_report(df_train_origin, df_test_origin, label, report_dst)
    df2 = _gen_eva_report(df_train_origin, df_test_origin, label, report_dst)
    df2.columns = df2.columns.droplevel(1)
    del df2['IV']
    df2.index.name = 'feature_name'
    df = df2.join(df1, how='outer').round(decimals)

    df = df.reset_index()
    tmp = df[['feature_name', 'IV']]
    tmp.columns = ['_'.join(col).strip() for col in tmp.columns.values]
    tmp = tmp[['feature_name_', 'IV_test']].drop_duplicates().sort_values('IV_test', ascending=False)
    columns = tmp.feature_name_

    df_res = pd.DataFrame()
    for c in columns:
        tmp = df[df['feature_name'] == c]
        try:
            tmp.level = tmp.level.replace('missing', np.nan).astype(float)
        except:
            pass
        tmp = tmp.sort_values([('IV', 'train'), 'feature_name', 'level'], ascending=[False, True, True])
        tmp.level = tmp.level.replace(np.nan, 'missing')
        tmp = tmp.set_index(['feature_name', 'level', 'bin'])
        df_res = pd.concat([df_res, tmp], axis=0)

    if gen_report:
        df_res.to_pickle(os.path.join(report_dst, 'woe_eva_report.report'))
    return df_res

def zeros_deal(test_value, train_value, smooth=0.01):
    difference_rate = 1 - (test_value + smooth) / (train_value + smooth)
    return difference_rate

def stable_different(df, process_name, threshold):
    df_diff = df[df['psi'] >= threshold]
    df_result = pd.DataFrame()
    if df_diff.empty == False:

        for feat_name in df_diff.index.tolist():
            df_result_m = pd.DataFrame()
            df_result_m['feature_name'] = [feat_name]
            df_result_m['process_name'] = process_name
            df_result_m['warning_indice'] = 'PSI'
            df_result_m['warning_reason'] = "数据集的PSI为" + str(df_diff[df_diff.index == feat_name].psi.values[0])
            df_result_m['detailed_data'] = "PSI : " + str(df_diff[df_diff.index == feat_name].psi.values[0])
            df_result = pd.concat([df_result_m, df_result], axis=0)
    return df_result

def woe_different(df, process_name, threshold, woe_columns=['KS', 'AUC', 'IV']):
    df_result_fin = pd.DataFrame()
    for woe_column in woe_columns:
        df[woe_column + '_difference_rate'] = zeros_deal(df[woe_column]['test'], df[woe_column]['train'], smooth=0.01)
        df_diff = df[
            (df[woe_column + '_difference_rate'] >= threshold) | (df[woe_column + '_difference_rate'] <= -threshold)]
        df_result = pd.DataFrame()
        if df_diff.empty == False:
            for feat_name, feat_name_df in df_diff.reset_index().groupby('feature_name'):
                df_result_m = pd.DataFrame()
                df_result_m['feature_name'] = [feat_name]
                df_result_m['process_name'] = process_name
                df_result_m['warning_indice'] = woe_column
                df_result_m['warning_reason'] = "train数据集和test数据集" + woe_column + "差异率为" + format(
                    feat_name_df[woe_column + '_difference_rate'].values[0], '.2%')
                df_result_m['detailed_data'] = woe_column + " train : " + str(
                    feat_name_df[woe_column]['train'].values[0]) + " " + woe_column + " test : " + str(
                    feat_name_df[woe_column]['test'].values[0])
                df_result = pd.concat([df_result_m, df_result], axis=0)
        df_result_fin = pd.concat([df_result, df_result_fin], axis=0)
        df_result_fin
    return df_result_fin

def desc_different(df, process_name, threshold, desc_columns=['cover', '50%']):
    df = df.sort_index(level='feature_name')
    df_result_fin = pd.DataFrame()
    for desc_column in desc_columns:
        diff_rate_m = pd.DataFrame(index=df.index)
        train_i = pd.Series(df.loc[(df.index.levels[0].tolist(), slice(None), ['train']), desc_column])
        test_i = pd.Series(df.loc[(df.index.levels[0].tolist(), slice(None), ['test']), desc_column])
        diff_rate_m['test_difference_rate'] = test_i
        diff_rate_m = diff_rate_m.fillna(method='ffill')
        diff_rate_m['train_difference_rate'] = train_i
        diff_rate_m = diff_rate_m.fillna(method='bfill')
        diff_rate_m[desc_column + '_difference_rate'] = zeros_deal(diff_rate_m['test_difference_rate'],
                                                                   diff_rate_m['train_difference_rate'])
        df_m = df.join(diff_rate_m, how='inner')

        df_diff = df_m[(df_m[desc_column + '_difference_rate'] >= threshold) | (
            df_m[desc_column + '_difference_rate'] <= -threshold)]
        df_result = pd.DataFrame()
        if df_diff.empty == False:
            for feat_name, feat_name_df in df_diff.groupby('feature_name'):
                df_result_m = pd.DataFrame()
                df_result_m['feature_name'] = [feat_name]
                df_result_m['process_name'] = process_name
                df_result_m['warning_indice'] = desc_column
                df_result_m['warning_reason'] = "train数据集和test数据集" + desc_column + "差异率为" + format(
                    feat_name_df[desc_column + '_difference_rate'][0], '.2%')

                df_result_m['detailed_data'] = desc_column + " train : " + str(
                    feat_name_df.loc[(feat_name_df.index.levels[0].tolist(), slice(None), ['train']), desc_column][
                        0]) + " " + desc_column + " test : " + str(
                    feat_name_df.loc[(feat_name_df.index.levels[0].tolist(), slice(None), ['test']), desc_column][0])
                df_result = pd.concat([df_result_m, df_result], axis=0)
        df_result_fin = pd.concat([df_result, df_result_fin], axis=0)

    return df_result_fin

def _gen_warning_report(report_src, report_dst, gen_report=True):
    if os.path.exists(report_src) == False:
        raise ValueError('%s is not exists!' % report_src)
    else:
        df_result = pd.DataFrame()
        for root, dirs, files in os.walk(report_src):
            if root != report_src:
                for file in files:
                    if file.endswith('report'):
                        path = os.path.join(root, file)
                        sheetname = os.path.basename(os.path.dirname(root)) + '_' + file[:-7]
                        if 'woe_eva' in file:
                            df_woe = pd.read_pickle(path)
                            df_diff_m = woe_different(df_woe, sheetname, 0.2)
                        elif 'stable_test' in file:
                            df_stable = pd.read_pickle(path)
                            df_diff_m = stable_different(df_stable, sheetname, 0.2)
                        elif 'data_stats' in file:
                            df_desc = pd.read_pickle(path)
                            df_diff_m = desc_different(df_desc, sheetname, 0.2)
                        else:
                            df_diff_m = pd.DataFrame()
                        df_result = pd.concat([df_diff_m, df_result], axis=0)
        df_result = df_result.sort_values('feature_name')
        df_result = df_result.reset_index(drop=True)
    if gen_report:
        if not os.path.exists(os.path.join(report_dst, 'reporter')):
            os.makedirs(os.path.join(report_dst, 'reporter'))
        df_result.to_pickle(os.path.join(report_dst, 'reporter', 'warning_report.report'))
    return df_result

def _gen_stable_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=True,
                       covariate_shift_eva=True):
    df_train_cp = df_train_origin.copy()
    df_test_cp = df_test_origin.copy()
    # stable_test
    dic, mcc, weights = dm.consistent_dataset_test(df_train_cp, df_test_cp, label, report_dst=None,
                                                   covariate_shift_eva=covariate_shift_eva)

    df = pd.DataFrame(dic).T
    df = pd.concat([df, pd.DataFrame({'mcc': {'All_Dataset': mcc}})], axis=0).round(decimals)

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'stable_test.report'))
        stable_test = {'single_feature_test': dic, 'mcc': mcc}
        simplejson.dump(stable_test, open(os.path.join(report_dst, 'stable_test.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
        pickle.dump(weights, open(os.path.join(report_dst, 'weights.pkl'), 'wb'))
    return df

def _gen_eda_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=True,
                    covariate_shift_eva=True):
    tmp = _gen_woe_eva_report(df_train_origin, df_test_origin, label, report_dst, decimals=decimals,
                              gen_report=gen_report)
    tmp = _gen_desc_report(df_train_origin, df_test_origin, label, report_dst, decimals=decimals, gen_report=gen_report)
    tmp = _gen_stable_report(df_train_origin, df_test_origin, label, report_dst, decimals=decimals,
                             gen_report=gen_report,
                             covariate_shift_eva=covariate_shift_eva)
    return

def reporter_report(report_src, report_dst=None):
    """

    :param report_dst: 工作流的根目录
    :return:
    """
    if report_dst is None:
        report_dst = os.path.join(report_src, 'reporter')
        if not os.path.exists(report_dst):
            os.makedirs(report_dst)
    tmp = _gen_warning_report(report_src, report_dst)
    merge_excel(report_src, os.path.join(report_dst, 'all_report.xlsx'))
    return

def sampler_report(sample_num, dic, group_ratio, report_dst):
    """
    报告分组基准，分组前后样本分布对比
    :param sample_num:
    :param dic:
    :param group_ratio:
    :param report_dst:
    :return:
    """
    simplejson.dump(sample_num, open(os.path.join(report_dst, 'sample_num.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)
    simplejson.dump(dic, open(os.path.join(report_dst, 'dic.json'), 'w'), ignore_nan=True, cls=simplejsonEncoder)
    simplejson.dump(group_ratio, open(os.path.join(report_dst, 'group_ratio.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

def spliter_report(X_train, X_test, time_col, label_col, report_dst, gen_report=True, covariate_shift_eva=True):
    """
    覆盖率、饱和度统计，数据描述性统计和直方图，目标分布统计
    :param df_train:
    :param df_test:
    :param time_col:
    :param label_col:
    :param report_dst:
    :return:
    """
    df_train = X_train.copy()
    df_test = X_test.copy()

    # # TODO: 等待删除，暂时前端需要该输出
    # c_tr = cover_stats(df_train)
    # c_te = cover_stats(df_test)
    # c_tr.to_pickle(os.path.join(report_dst, 'c_tr.pkl'))
    # c_te.to_pickle(os.path.join(report_dst, 'c_te.pkl'))
    #
    # dic = {'train': table_to_dict(c_tr), 'test': table_to_dict(c_te)}
    # simplejson.dump(dic, open(os.path.join(report_dst, 'cover_stats.json'), 'w'), ignore_nan=True,
    #                 cls=simplejsonEncoder)

    d_tr = target_stats(df_train, label_col, time_col)
    d_te = target_stats(df_test, label_col, time_col)
    dic = {'train': d_tr, 'test': d_te}
    df = pd.DataFrame(dic).T
    df.to_pickle(os.path.join(report_dst, 'target_stats.report'))
    simplejson.dump(dic, open(os.path.join(report_dst, 'target_stats.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

    del df_train[time_col], df_test[time_col]

    _gen_eda_report(df_train, df_test, label_col, report_dst, gen_report=gen_report,
                    covariate_shift_eva=covariate_shift_eva)

def transformer_report(df_train_origin, df_test_origin, label, report_dst, gen_report=True, covariate_shift_eva=True):
    """
    单变量分析、直方排序图分析、train/test特征一致性分布分析
    :param df_train:
    :param df_test:
    :param label:
    :param report_dst:
    :return:
    """
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()

    _gen_eda_report(df_train_origin, df_test_origin, label, report_dst, gen_report=gen_report,
                    covariate_shift_eva=covariate_shift_eva)

    # feature_plot
    df_train.fillna(-999, inplace=True)
    train_plot_data = feature_plot(df_train, label, path=os.path.join(report_dst, 'train_feature_plot.png'))
    df_test.fillna(-999, inplace=True)
    test_plot_data = feature_plot(df_test, label, path=os.path.join(report_dst, 'test_feature_plot.png'))

    train_plot_data = {k: {'train': v} for k, v in train_plot_data.items()}
    test_plot_data = {k: {'test': v} for k, v in test_plot_data.items()}
    plot_data = defaultdict(dict)
    for k, v in chain(train_plot_data.items(), test_plot_data.items()):
        plot_data[k].update(v)
    simplejson.dump(plot_data, open(os.path.join(report_dst, 'plot_data.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

def filter_report(enc, df_train, df_test, label, report_dst, gen_report=True, covariate_shift_eva=True):
    """
    过滤指标得分输出
    :param enc:
    :param score_func:
    :param report_dst:
    :return:
    """
    data = {}
    for encoder_dict in enc.st:
        df = encoder_dict['enc'].features_report
        data[encoder_dict['method']] = table_to_dict(df)

    simplejson.dump(data, open(os.path.join(report_dst, 'feature_indices.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

    df = pd.DataFrame()
    for encoder_dict in enc.st:
        t = encoder_dict['enc'].features_report
        t.columns = pd.MultiIndex.from_product([[encoder_dict['method']], t.columns])
        df = pd.concat([df, t], axis=1)

    t = pd.Series(True, index=df.index)
    for c in df.columns:
        if c[-1].endswith('_support'):
            t = t & df[c]
    df['final_supprt'] = t

    df.round(4).to_pickle(os.path.join(report_dst, 'feature_filter.report'))

    _gen_eda_report(df_train, df_test, label, report_dst, gen_report=gen_report,
                    covariate_shift_eva=covariate_shift_eva)

def optimizer_report(all_df, report_dst):
    """
    每轮迭代参数和效果的输出
    :param all_df:
    :param report_dst:
    :return:
    """
    all_df.to_pickle(os.path.join(report_dst, 'optimizer_result.report'))
    """
    Currently (as of Pandas version 0.18), df.to_dict('records') accesses the NumPy array df.values.
    This property upcasts the dtype of the int column to float so that the array can have a single common dtype.
    After this point there is no hope of returning the desired result -- all the ints have been converted to floats.
    refers: https://stackoverflow.com/questions/37897527/get-python-pandas-to-dict-with-orient-records-but-without-float-cast
    """
    dic = [{col: getattr(row, col) for col in all_df} for row in all_df.itertuples()]
    simplejson.dump(dic, open(os.path.join(report_dst, 'all_params_result.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

def trainer_report(enc, df_train, df_val, test_src, test_data_dst, report_dst):
    """
    训练报告
    :param enc:
    :param df_train:
    :param df_val:
    :param df_test:
    :param test_m:
    :param report_dst:
    :return:
    """
    if test_src is not None:
        df_test = enc.transform(test_src)
        df_test.to_pickle(test_data_dst)
        test_m = 'oot'
    else:
        test_m = 'test'

    df = get_data(test_src)

    try:
        df_label = df.pop(enc.label)
        df_label.name = 'y_true'
    except:
        df_label = pd.DataFrame()

    if enc.enc._estimator_type == 'ruler':
        tmp = enc.enc.get_metrics(df, df_label)
        tmp.round(4).to_pickle(os.path.join(report_dst, 'rule_report.report'))
        tmp = table_to_dict(tmp)
        simplejson.dump(tmp, open(os.path.join(report_dst, 'rule_report.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)

    try:
        rr = feature_evaluation.ResultReport()
        rr.fit(df_train.y_pred)
        df = rr.transform({'train': df_train, 'val': df_val, test_m: df_test})
        df.to_pickle(os.path.join(report_dst, 'level_report.report'))
        pickle.dump(rr, open(os.path.join(report_dst, 'rr_enc.pkl'), 'wb'))

        eval_plot_path = os.path.join(report_dst, 'trainer_eva_report.png')
        dic_all = eva_plot({'train': [df_train.y_true, df_train.y_pred], 'val': [df_val.y_true, df_val.y_pred],
                            test_m: [df_test.y_true, df_test.y_pred]}, path=eval_plot_path)
        import re
        from collections import defaultdict
        dic = defaultdict(dict)
        token = re.compile('[\d.]+')
        for k in dic_all['roc_curve']:
            for v in dic_all['ks_curve'][k]:
                if v.startswith('KS'):
                    s = float(re.findall(token, v)[0])
                    #             dic[k].append({'ks': s})
                    dic[k]['ks'] = s
        for v in dic_all['roc_curve']['train'].keys():
            k1, k2 = v.split('(')
            k1 = k1.strip()
            s = float(re.findall(token, k2)[0])
            #     dic[k1].append({'auc': s})
            dic[k1]['auc'] = s
        # df = pd.DataFrame.from_dict(dic)
        simplejson.dump(dict(dic), open(os.path.join(report_dst, 'abstract_data.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
        simplejson.dump(dic_all, open(os.path.join(report_dst, 'eva_data.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)

        pd.DataFrame(dic).T.to_pickle(os.path.join(report_dst, 'abstract_data.report'))

        importances = getattr(enc.enc, "feature_importances_", None)
        coef_ = getattr(enc.enc, "coef_", None)
        rules_ = getattr(enc.enc, "rules_", None)
        if not (importances is None and coef_ is None):
            try:
                df_features = \
                    pd.DataFrame(
                        {'feature_names': enc.columns, 'feature_importances': get_feature_importances(enc.enc)})[
                        ['feature_names', 'feature_importances']]
            except Exception as e:
                print(traceback.format_exc())
                df_features = pd.DataFrame()
        elif rules_ is not None:
            try:
                df_features = pd.DataFrame({'rules': enc.enc.rules_})
            except Exception as e:
                print(traceback.format_exc())
                df_features = pd.DataFrame()
        else:
            df_features = pd.DataFrame()

        simplejson.dump(df_features.to_dict(orient='records'),
                        open(os.path.join(report_dst, 'feature_importances.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)

        pd.DataFrame(df_features).round(4).to_pickle(os.path.join(report_dst, 'feature_importances.report'))
    except Exception as e:
        print(traceback.format_exc())

    report_src = os.path.dirname(os.path.dirname(report_dst))
    reporter_report(report_src)
    return


# flake8: noqa

# Copyright (c) 2016 Tesora, Inc.
#
# This file is part of the Tesora DBaas Platform Enterprise Edition.
#
# Tesora DBaaS Platform is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
# for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# According to sec. 7 of the GNU Affero General Public License, version 3, the
# terms of the AGPL are supplemented with the following terms:
#
# "Tesora", "Tesora DBaaS Platform", and the Tesora logo are trademarks
#  of Tesora, Inc.,
#
# The licensing of the Program under the AGPL does not imply a trademark
# license. Therefore any rights, title and interest in our trademarks remain
# entirely with us.
#
# However, if you propagate an unmodified version of the Program you are
# allowed to use the term "Tesora" solely to indicate that you distribute the
# Program. Furthermore you may use our trademarks where it is necessary to
# indicate the intended purpose of a product or service provided you use it in
# accordance with honest practices in industrial or commercial matters.
#
# If you want to propagate modified versions of the Program under the name
# "Tesora" or "Tesora DBaaS Platform", you may only do so if you have a written
# permission by Tesora, Inc. (to acquire a permission please contact
# Tesora, Inc at trademark@tesora.com).
#
# The interactive user interface of the software displays an attribution notice
# containing the term "Tesora" and/or the logo of Tesora.  Interactive user
# interfaces of unmodified and modified versions must display Appropriate Legal
# Notices according to sec. 5 of the GNU Affero General Public License,
# version 3, when you propagate unmodified or modified versions of  the
# Program. In accordance with sec. 7 b) of the GNU Affero General Public
# License, version 3, these Appropriate Legal Notices must retain the logo of
# Tesora or display the words "Initial Development by Tesora" if the display of
# the logo is not reasonably feasible for technical reasons.

from eventlet.timeout import Timeout
from oslo_log import log as logging
from oslo_utils import importutils

from trove.common import cfg
from trove.common import exception
from trove.common.i18n import _
from trove.common.strategies.cluster import base
from trove.common.strategies.cluster.oracle_rac import utils as rac_utils
from trove.common import utils
from trove.extensions.security_group import models as secgroup_models
from trove.instance import models as instance_models
from trove.taskmanager import api as task_api
from trove.taskmanager import models as task_models

LOG = logging.getLogger(__name__)
CONF = cfg.CONF
USAGE_SLEEP_TIME = CONF.usage_sleep_time  # seconds.

class OracleRACTaskManagerStrategy(base.BaseTaskManagerStrategy):

    @property
    def task_manager_api_class(self):
        return OracleRACTaskManagerAPI

    @property
    def task_manager_cluster_tasks_class(self):
        return OracleRACClusterTasks

class OracleRACClusterTasks(task_models.ClusterTasks):

    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s." % cluster_id)
        self.ds_conf = CONF.get(self.datastore_version.manager)
        self.network_driver = (importutils.import_class(
            CONF.network_driver))(context, None)

        def _create_cluster():
            db_instances = instance_models.DBInstance.find_all(
                cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]
            LOG.debug("instances in cluster %s: %s" % (cluster_id,
                                                       instance_ids))
            if not self._all_instances_ready(instance_ids, cluster_id):
                return
            LOG.debug("all instances in cluster %s ready." % cluster_id)
            instances = [instance_models.Instance.load(
                context, instance_id) for instance_id in instance_ids]

            self._open_firewall_between_nodes(instance_ids)
            self._get_subnets()

            instance_guests = [self.get_guest(instance)
                               for instance in instances]

            for instance_guest in instance_guests:
                instance_guest.configure_hosts(
                    self.name,
                    self.pub_subnet['cidr'],
                    self.int_subnet['cidr'])
            self._get_hostnames(instances)
            for instance_guest in instance_guests:
                instance_guest.establish_ssh_user_equivalency(
                    self.host_ip_pairs)

            instance_guests[0].configure_grid(
                self.id, self.name, self._make_nodes_string(include_vips=True),
                self.pub_subnet['cidr'], self.int_subnet['cidr'])
            for instance_guest in instance_guests:
                instance_guest.run_grid_root()

            instance_guests[0].install_oracle_database(
                self._make_nodes_string())
            for instance_guest in instance_guests:
                instance_guest.run_oracle_root()

            self._allow_interconnect_vips(instances, instance_guests)

            instance_guests[0].create_rac_database(self._make_nodes_string())

            # for instance_guest in instance_guests[1:]:
            #     instance_guest.determine_sid()

            for instance in instances:
                self.get_guest(instance).cluster_complete()

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s." % cluster_id)

    def _open_firewall_between_nodes(self, instance_ids):
        LOG.debug("Opening firewall between cluster instances.")
        security_groups = [
            secgroup_models.SecurityGroup.
                get_security_group_by_id_or_instance_id(instance_id,
                                                        self.tenant_id)
            for instance_id in instance_ids]
        for security_group in security_groups:
            for friendly_group in security_groups:
                if security_group == friendly_group.id:
                    # same group
                    continue
                self.network_driver.create_security_group_rule(
                    security_group.id, ethertype='IPv4',
                    direction='ingress', remote_group_id=friendly_group.id)

    def _get_subnets(self):
        public_subnet_name = rac_utils.make_object_name(
            self.ds_conf, ['public', 'subnet'], self.id)
        interconnect_subnet_name = rac_utils.make_object_name(
            self.ds_conf, ['interconnect', 'subnet'], self.id)
        subnets = self.network_driver.list_subnets()
        self.pub_subnet = None
        self.int_subnet = None
        for subnet in subnets:
            if subnet['name'] == public_subnet_name:
                self.pub_subnet = subnet
            elif subnet['name'] == interconnect_subnet_name:
                self.int_subnet = subnet
        if not self.pub_subnet and self.int_subnet:
            raise exception.TroveError(_(
                "Could not find cluster's public and interconnect subnets."))
        LOG.debug("Cluster {clu} is using public subnet {pubsub} and "
                  "interconnect subnet {intsub}".format(
                      clu=self.id, pubsub=self.pub_subnet['cidr'],
                      intsub=self.int_subnet['cidr']))

    def _get_hostnames(self, instances):
        self.host_ip_pairs = []
        pub_subnet_manager = rac_utils.RACPublicSubnetManager(
            self.pub_subnet['cidr'])
        network_name = self.network_driver.get_network_by_id(
            self.pub_subnet['network_id'])['name']
        for instance in instances:
            public_ip = instance.addresses[network_name][0]['addr']
            i = pub_subnet_manager.instance_index_from_ip(public_ip)
            hostname = rac_utils.make_instance_hostname(self.name, i)
            # append a tuple
            self.host_ip_pairs.append((hostname, public_ip))

    def _make_nodes_string(self, include_vips=False):
        nodes = []
        for pair in self.host_ip_pairs:
            hostname = pair[0]
            if include_vips:
                nodes.append(':'.join([hostname, hostname + '-vip']))
            else:
                nodes.append(hostname)
        return ','.join(nodes)

    def _allow_interconnect_vips(self, instances, instance_guests):
        network_name = rac_utils.make_interconnect_network_name(self.ds_conf,
                                                                self.id)
        # get the interconnect subnet's ports
        ports = [port for port in self.network_driver.list_ports()
                 if port['fixed_ips'][0]['subnet_id'] == self.int_subnet['id']]
        for i in range(len(instances)):
            ip = instances[i].addresses[network_name][0]['addr']
            vip = instance_guests[i].get_private_vip(ip)
            port_id = [port for port in ports
                       if port['fixed_ips'][0]['ip_address'] == ip][0]['id']
            self.network_driver.update_port(
                port_id, allowed_address_pairs=[{'ip_address': vip}])

class OracleRACTaskManagerAPI(task_api.API):
    pass


# coding: utf-8
import RPi.GPIO as GPIO
import time
from datetime import datetime
import argparse
import csv

def readadc(adcnum, clockpin, mosipin, misopin, cspin):
    if adcnum > 7 or adcnum < 0:
        return -1

    GPIO.output(cspin, GPIO.HIGH)
    GPIO.output(clockpin, GPIO.LOW)
    GPIO.output(cspin, GPIO.LOW)

    commandout = adcnum
    commandout |= 0x18
    commandout <<= 3
    for i in range(5):
        if commandout & 0x80:
                GPIO.output(mosipin, GPIO.HIGH)
        else:
                 GPIO.output(mosipin, GPIO.LOW)
        commandout <<= 1
        GPIO.output(clockpin, GPIO.HIGH)
        GPIO.output(clockpin, GPIO.LOW)
    adcout = 0
    for i in range(11):
        GPIO.output(clockpin, GPIO.HIGH)
        GPIO.output(clockpin, GPIO.LOW)
        adcout <<= 1
        if i>0 and GPIO.input(misopin)==GPIO.HIGH:
            adcout |= 0x1
    GPIO.output(cspin, GPIO.HIGH)
    return adcout

def init_adc(SPICLK, SPIMOSI, SPIMISO, SPICS, READPIN):
    GPIO.setup(SPICLK, GPIO.OUT)
    GPIO.setup(SPIMOSI, GPIO.OUT)
    GPIO.setup(SPIMISO, GPIO.IN)
    GPIO.setup(SPICS, GPIO.OUT)
    GPIO.setup(READPIN, GPIO.OUT)
    GPIO.output(READPIN, GPIO.HIGH)

def main(args):
    GPIO.setmode(GPIO.BCM)
    SPICS = 8
    SPIMISO = 9
    SPIMOSI = 10
    SPICLK = 11
    READPIN = 25
    channel = 0
    init_adc(SPICLK, SPIMOSI, SPIMISO, SPICS, READPIN)

    fname = args.name
    run_time = args.time
    sleep_time = 1 / args.hz

    dir_ = r'./data/raw/'
    f = open(dir_+args.name+'.csv', 'a', newline='')
    writer = csv.writer(f)
    try:
        t_s = time.time()
        t = 0
        while t < run_time:
            t = time.time() - t_s
            value = readadc(channel, SPICLK, SPIMOSI, SPIMISO, SPICS)
            writer.writerow([t, value])
            print('time: {:.2f}/{}\t\tvalue: {}'.format(t, run_time, value))
            time.sleep(sleep_time)

    except KeyboardInterrupt as e:
        print(e)

    finally:
        print('finish.')
        f.close()
        GPIO.cleanup()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--name',type=str, default=datetime.now().isoformat())
    parser.add_argument('-t', '--time', type=int, default=60)
    parser.add_argument('--hz', type=int, default=2)
    args = parser.parse_args()
    main(args)


#!/usr/bin/env python3
# HW05_ex00_logics.py

##############################################################################
def even_odd():
    """ Print even or odd:
        Takes one integer from user
            accepts only non-word numerals
            must validate
        Determines if even or odd
        Prints determination
        returns None
    """
    user_number = ''
    user_number = input('What is your number?')
    while True:
        try:
            while user_number != '':
                user_number = int(user_number)
                if user_number%2 == 0:
                    print("Your number is even")
                    return
                else:
                    print("Your number is odd")
                    return
            break
        except:
            print("Only use integers!")
            even_odd()
            return

def ten_by_ten():
    """ Prints integers 1 through 100 sequentially in a ten by ten grid."""

    line_start = int(1)
    line_stop = int(11)
    while line_stop <= 101:
        for numbers in range (line_start, line_stop):
            print ('{:3}'.format(numbers), end=' ') #found this {:3}.format on SO, and don't completely understand it. I assume it tells the number to print on the 3rd position on the line.
        print("\n")
        line_start += 10
        line_stop += 10
    return

    # I just want to point out all the crazy stuff
    # I tried before I figured this one out. At least
    #  I think I figured it out.

    # count = ''
    # number = 1
    # while number <= 10:
    #    print(number, " ")
    #    number += 1
    # # for x in range (1, 101):
    # #     print (x+0, end=' ')
    # # print()

    # for numbers in range (11, 21):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (21, 31):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (31, 41):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (41, 51):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (51, 61):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (61, 71):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (71, 81):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (91, 101):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")

    # for i in     for numbers in range (1, 11):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")
    # for numbers in range (11, 21):
    #     print ('{:3}'.format(numbers), end=' ')
    # print("\n")range(11, 101):
    # #     print(i, end = ' ')
    #     # if i == len(range(1,11)):print('{:3}'.format(i), end=' ')

    #     if i == len(range(1,21)):print()
    #     if i == len(range(1,31)):print()
    #     if i == len(range(1,41)):print()
    #     if i == len(range(1,51)):print()
    #     if i == len(range(1,61)):print()
    #     if i == len(range(1,71)):print()
    #     if i == len(range(1,81)):print()
    #     if i == len(range(1,91)):print()

from statistics import mean

def find_average():
    """ Takes numeric input (non-word numerals) from the user, one number
    at a time. Once the user types 'done', returns the average.
    """
    numeric_input = ''
    counter = 1
    found_average = ''
    input_sum = ''
    inputted_numbers = []

    while True:
        try:
            while numeric_input != 'done':
                numeric_input = input("What is your input?")
                inputted_numbers.append(int(numeric_input))
                if numeric_input == 'done':
                    print("You're done!")
        except:
             print('Only use numeric values!')

        print("We found the average:", mean(inputted_numbers))
        break

    return
##############################################################################
def main():
    even_odd()
    ten_by_ten()
    find_average()

if __name__ == '__main__':
    main()