# uncompyle6 version 3.7.4
# Python bytecode 3.4 (3310)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/nazrul/www/python/Contributions/apps/hybrid-access-control-system/hacs/management/commands/export_routes.py
# Compiled at: 2016-06-19 12:45:26
# Size of source mod 2**32: 14868 bytes
import os, time, json, shutil
from django.utils import six
from django.apps import apps
from django.db.models import Q
from django.conf import settings
from django.core import serializers
from django.contrib.sites.models import Site
from django.contrib.auth.models import Group
from django.utils.encoding import smart_text
from django.contrib.auth import get_user_model
from django.core.management import BaseCommand
from django.core.management import CommandError
from django.utils.module_loading import import_string
from django.contrib.contenttypes.models import ContentType
from django.core.serializers.json import DjangoJSONEncoder
from hacs.models import RoutingTable
from hacs.models import SiteRoutingRules
from hacs.models import ContentTypeRoutingRules
from hacs.defaults import HACS_SERIALIZED_ROUTING_DIR
from hacs.defaults import HACS_USER_OBJECT_QUERY_CALLABLE
__author__ = 'Md Nazrul Islam<connect2nazrul@gmail.com>'
UserModel = get_user_model()

class Command(BaseCommand):
    __doc__ = ' '
    help = 'HACS: Command tool for exporting'
    can_import_settings = True

    def add_arguments(self, parser):
        """"""
        parser.add_argument('-d', '--destination', action='store', dest='destination', help='The destination where serialized data will be stored physically. Value could be directory or absolute file path')
        parser.add_argument('-a', '--exclude-apps', action='store', dest='exclude_apps', nargs='+', help='List of apps you want to ignore. Example: --exclude-apps app1 app2 ..')
        parser.add_argument('-s', '--exclude-sites', action='store', dest='exclude_sites', nargs='+', help='List of sites you want to ignore. Value must be `domain name` of site not ID Example: --exclude-sites site1 site2 ..')
        parser.add_argument('-g', '--exclude-groups', action='store', dest='exclude_groups', nargs='+', help='List of user groups you want to ignore. Value must group name not ID. Example: --exclude-groups group1 group2 ..')
        parser.add_argument('-u', '--exclude-users', action='store', dest='exclude_users', nargs='+', help='List of users you want to ignore. Value must username or email not ID. Example: --exclude-users user1 user2 ..')
        parser.add_argument('-r', '--exclude-routes', action='store', dest='exclude_routes', nargs='+', help='List of routes you want to ignore. Value must provide route name not ID. Example: --exclude-routes route1 route2 ..')
        parser.add_argument('-p', '--print-only', action='store_true', dest='print_only', help='Use this option if you want to just print serialized data instead of store to filesystem')
        parser.add_argument('-x', '--extended-natural-keys', action='store_true', dest='extended_natural_keys', help='Enable extended natural keys.I.e for Site, objects_is')
        parser.add_argument('-f', '--format', action='store', dest='output_format', choices=('json',
                                                                                'xml'), default='json', help='Output format of serialized data or type of serializer. by default is json')
        parser.add_argument('--no-natural-primary', action='store_true', dest='no_natural_primary_keys', help='By default this tool uses natural primary keys, but you can omit by using this option')
        parser.add_argument('--no-natural-foreign', action='store_true', dest='no_natural_foreign_keys', help='By default this tool uses natural foreign keys, but you can omit by using this option')

    def handle(self, *args, **options):
        """"""
        self._validate(*args, **options)
        objects = self.collect_all_objects_list(*args, **options)
        if not objects:
            self.stdout.write('Nothing to export. Empty result.')
            return
        filename, serializer = self._calculate_output_filename(*args, **options)
        serializer_options = {}
        if not options['no_natural_primary_keys']:
            serializer_options['use_natural_primary_keys'] = True
        if not options['no_natural_foreign_keys']:
            serializer_options['use_natural_foreign_keys'] = True
        if filename:
            with open(filename, 'w') as (f):
                serializers.serialize(serializer, objects, stream=f, **serializer_options)
            if options['extended_natural_keys']:
                if serializer == 'json':
                    with open(filename, 'r') as (f):
                        objects = json.load(f, encoding=settings.DEFAULT_CHARSET)
                        self._wrap_extended_natural_keys(objects)
                        with open(filename + '.tmp', 'w') as (fp):
                            json.dump(objects, fp=fp, cls=DjangoJSONEncoder)
                    shutil.move(filename + '.tmp', filename)
        else:
            data = serializers.serialize(serializer, objects, **serializer_options)
            if options['extended_natural_keys']:
                if serializer == 'json':
                    data = json.loads(data)
                    self._wrap_extended_natural_keys(data)
                    data = json.dumps(data, cls=DjangoJSONEncoder)
            self.stdout.write(data)

    def collect_all_objects_list(self, *args, **kwargs):
        """"""
        filters = {}
        exclude_routes = kwargs['exclude_routes']
        if exclude_routes is not None:
            if isinstance(exclude_routes, six.string_types):
                exclude_routes = (
                 exclude_routes,)
        if exclude_routes:
            filters['route_name__in'] = exclude_routes
        routing_table_queryset = RoutingTable.objects.exclude(**filters)
        if filters.get('route_name__in', None):
            filters['route__in'] = [RoutingTable.objects.get_by_natural_key(x) for x in filters.get('route_name__in')]
            del filters['route_name__in']
        exclude_sites = kwargs['exclude_sites']
        if exclude_sites is not None:
            if isinstance(exclude_sites, six.string_types):
                exclude_sites = (
                 exclude_sites,)
        if exclude_sites:
            filters['site__in'] = [Site.objects.get(domain=x) for x in exclude_sites]
        site_routing_queryset = SiteRoutingRules.objects.exclude(**filters)
        filter_arg = ()
        exclude_groups = kwargs['exclude_groups']
        if exclude_groups is not None:
            if isinstance(exclude_groups, six.string_types):
                exclude_groups = (
                 exclude_groups,)
        if exclude_groups:
            exclude_groups = [Group.objects.get_by_natural_key(x).pk for x in exclude_groups]
            filter_arg = (Q(content_type=ContentType.objects.get_for_model(Group), object_id__in=exclude_groups),)
        exclude_users = kwargs['exclude_users']
        if exclude_users is not None:
            if isinstance(exclude_users, six.string_types):
                exclude_users = (
                 exclude_users,)
        if exclude_users:
            exclude_users = [import_string(getattr(settings, 'HACS_USER_OBJECT_QUERY_CALLABLE', HACS_USER_OBJECT_QUERY_CALLABLE))(x, silent=False).pk for x in exclude_users]
            if filter_arg:
                filter_arg = (filter_arg[0] | Q(content_type=ContentType.objects.get_for_model(UserModel), object_id__in=exclude_users),)
            else:
                filter_arg = (
                 Q(content_type=ContentType.objects.get_for_model(UserModel), object_id__in=exclude_users),)
        contenttype_routing_queryset = ContentTypeRoutingRules.objects.exclude(*filter_arg, **filters)
        return list(routing_table_queryset) + list(site_routing_queryset) + list(contenttype_routing_queryset)

    def _validate(self, *args, **kwargs):
        """
        :param args:
        :param kwargs:
        :return:
        """
        if not kwargs['destination']:
            if not kwargs['print_only']:
                raise CommandError('You have to pass either destination of output file or enable print only')
        if kwargs['destination']:
            if '/' not in kwargs['destination']:
                if os.path.splitext(kwargs['destination'])[1]:
                    if not getattr(settings, 'HACS_SERIALIZED_ROUTING_DIR', HACS_SERIALIZED_ROUTING_DIR):
                        raise CommandError('You provide destination file name, not with full path, in that case you must define `HACS_SERIALIZED_ROUTING_DIR` at django settings')
        exclude_apps = kwargs['exclude_apps']
        if exclude_apps is not None:
            if isinstance(exclude_apps, six.string_types):
                exclude_apps = (
                 exclude_apps,)
        if exclude_apps:
            for app_label in exclude_apps:
                try:
                    apps.get_app_config(app_label)
                except LookupError as err:
                    raise CommandError('Invalid app: ' + smart_text(err))

        exclude_sites = kwargs['exclude_sites']
        if exclude_sites is not None:
            if isinstance(exclude_sites, six.string_types):
                exclude_sites = (
                 exclude_sites,)
        if exclude_sites:
            for site in exclude_sites:
                try:
                    Site.objects.get(domain=site)
                except Site.DoesNotExist:
                    raise CommandError("Specified site `%s` doesn't exists!" % site)

        exclude_groups = kwargs['exclude_groups']
        if exclude_groups is not None:
            if isinstance(exclude_groups, six.string_types):
                exclude_groups = (
                 exclude_groups,)
        if exclude_groups:
            for group in exclude_groups:
                try:
                    Group.objects.get_by_natural_key(group)
                except Group.DoesNotExist:
                    raise CommandError("Specified group `%s` doesn't exists!" % group)

        exclude_users = kwargs['exclude_users']
        if exclude_users is not None:
            if isinstance(exclude_users, six.string_types):
                exclude_users = (
                 exclude_users,)
        if exclude_users:
            for user in exclude_users:
                try:
                    import_string(getattr(settings, 'HACS_USER_OBJECT_QUERY_CALLABLE', HACS_USER_OBJECT_QUERY_CALLABLE))(user, silent=False)
                except UserModel.DoesNotExist:
                    raise CommandError("Specified user `%s` doesn't exists!" % user)

        exclude_routes = kwargs['exclude_routes']
        if exclude_routes is not None:
            if isinstance(exclude_routes, six.string_types):
                exclude_routes = (
                 exclude_routes,)
        if exclude_routes:
            for route in exclude_routes:
                try:
                    RoutingTable.objects.get_by_natural_key(route)
                except RoutingTable.DoesNotExist:
                    raise CommandError("Specified route `%s` doesn't exists!" % route)

        if kwargs['extended_natural_keys']:
            if kwargs['no_natural_primary_keys'] or kwargs['no_natural_foreign_keys']:
                raise CommandError('--extended-natural-keys could be only applied, if enable natural PK and FK')

    def _calculate_output_filename(self, *args, **kwargs):
        """ """
        if kwargs['print_only']:
            return (
             None, kwargs['output_format'])
        filename = kwargs['destination']
        if filename:
            if '/' not in filename and os.path.splitext(filename)[1]:
                filename = os.path.join(getattr(settings, 'HACS_SERIALIZED_ROUTING_DIR', HACS_SERIALIZED_ROUTING_DIR), filename)
        else:
            if '/' in filename:
                if not os.path.splitext(filename)[1]:
                    filename = os.path.join(filename, 'hacs_serialized_routes_generated_%s.%s' % (
                     time.time(), kwargs['output_format']))
            return (
             filename, kwargs['output_format'])

    def _wrap_extended_natural_keys(self, entries):
        """"""
        _allowed_models = (
         '%s.%s' % (SiteRoutingRules._meta.app_label, SiteRoutingRules._meta.model_name),
         '%s.%s' % (ContentTypeRoutingRules._meta.app_label, ContentTypeRoutingRules._meta.model_name))
        for entry in entries:
            if entry['model'].lower() not in _allowed_models:
                continue
            entry['fields']['site'] = (
             Site.objects.get(pk=entry['fields']['site']).domain,)
            if entry['model'].lower() == _allowed_models[1]:
                if UserModel._meta.app_label in entry['fields']['content_type'] and UserModel._meta.model_name in entry['fields']['content_type']:
                    entry['fields']['object_id'] = (getattr(UserModel.objects.get(pk=entry['fields']['object_id']), UserModel.USERNAME_FIELD),)
                else:
                    if Group._meta.app_label in entry['fields']['content_type']:
                        if Group._meta.model_name in entry['fields']['content_type']:
                            entry['fields']['object_id'] = Group.objects.get(pk=entry['fields']['object_id']).natural_key()
                    else:
                        continue


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.3'
#       jupytext_version: 1.0.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# 機械学習において、単一の学習器をそのまま使うのではなく、複数の学習器を組み合わせることで、予測エラーを小さくする手法をアンサンブル学習といいます。
# アンサンブル(混合学習手法の)には複数の学習器の平均や多数決を取るvoting、構成済の学習器の誤りを反映して次段の弱学習器を形成するboosting、そして初段の学習器の出力結果を次段の入力結果とするstacking(以下スタッキング)とよばれるものがあり、Kaggleなどのデータ分析コンペでは良く使われます。

# +
import pandas as pd
import numpy as np
import re
import sklearn
import xgboost as xgb # 勾配ブースティング
import seaborn as sns # オシャレなグラフ
import matplotlib.pyplot as plt
# %matplotlib inline

import plotly
import plotly.offline as py
py.init_notebook_mode(connected=True)
import plotly.graph_objs as go
import plotly.tools as tls

import warnings
warnings.filterwarnings('ignore')

# Going to use these 5 base models for the stacking
from sklearn.ensemble import (RandomForestClassifier, AdaBoostClassifier,
                              GradientBoostingClassifier, ExtraTreesClassifier)
from sklearn.svm import SVC
# from sklearn.cross_validation import KFold　バージョン古い書き方
from sklearn.model_selection import KFold
# -

# # Feature Exploration, Engineering and Cleaning
# ここでは、データを整形して特徴データを作成していきます。

# +
train = pd.read_csv('../Python_DataScience/lec56_train.csv')
test = pd.read_csv('./test.csv')

#乗客のID
PassengerId = test['PassengerId']

#最初の３行
train.head(3)
# -

# 使用するデータは、タイタニック号事件でのデータです。
#
# PassengerId – 乗客識別ユニークID
# Survived – 生存フラグ（0=死亡、1=生存）
# Pclass – チケットクラス
# Name – 乗客の名前
# Sex – 性別（male=男性、female＝女性）
# Age – 年齢
# SibSp – タイタニックに同乗している兄弟/配偶者の数
# parch – タイタニックに同乗している親/子供の数
# ticket – チケット番号
# fare – 料金
# cabin – 客室番号
# Embarked – 出港地（タイタニックへ乗った港）

# Feature Engineering
#
# 長い作業になりますが、機械学習において前処理が一番重要です。
#
# 前処理では、欠損値や文字データを全て数値データに変換します。

# +
full_data = [train, test]

#乗客の名前の長さ
train['Name_length'] = train['Name'].apply(len)
test['Name_length'] = test['Name'].apply(len)
# -

# 客室番号データがあるなら１を、欠損値なら0を
train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x) == float else 1)
test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x) == float else 1)

# 家族の大きさを"タイタニックに同乗している兄弟/配偶者の数"と
# "タイタニックに同乗している親/子供の数"から定義
for dataset in full_data:
    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1

# 家族がいるかどうか
# いるなら"IsAlone"が１
for dataset in full_data:
    dataset['IsAlone'] = 0
    dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1

# +
# 出港地の欠損値を一番多い"S"としておく
for dataset in full_data:
    dataset['Embarked'] = dataset['Embarked'].fillna('S')

# 料金の欠損値を中央値としておく
# 料金の大きく４つのグループに分ける
for dataset in full_data:
    dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
train['CategoricalFare'] = pd.qcut(train['Fare'], 4)

# 年齢を５つのグループに分ける
for dataset in full_data:
    age_avg = dataset['Age'].mean()
    age_std = dataset['Age'].std()
    age_null_count = dataset['Age'].isnull().sum()
    age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
    dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
    dataset['Age'] = dataset['Age'].astype(int)
train['CategoricalAge'] = pd.cut(train['Age'], 5)

# 名前を取り出す関数１
def get_title(name):
    title_search = re.search(' ([A-Za-z]+)\.', name)
    # 名前があれば取り出して返す
    if title_search:
        return title_search.group(1)
    return ""

# 関数１を使う
for dataset in full_data:
    dataset['Title'] = dataset['Name'].apply(get_title)

# 名前の変なところを変換
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')

    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')

for dataset in full_data:
    # 女なら０、男なら１
    dataset['Sex'] = dataset['Sex'].map( {'female': 0, 'male': 1} ).astype(int)

    # 名前の5種類にラベル付
    title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
    dataset['Title'] = dataset['Title'].map(title_mapping)
    dataset['Title'] = dataset['Title'].fillna(0)

    # 出港地の3種類にラベル付
    dataset['Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2} ).astype(int)

    # 料金を４つのグループに分ける
    dataset.loc[ dataset['Fare'] <= 7.91, 'Fare']                               = 0
    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1
    dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare']   = 2
    dataset.loc[ dataset['Fare'] > 31, 'Fare']                                  = 3
    dataset['Fare'] = dataset['Fare'].astype(int)

    # 年齢を5つのグループに分ける
    dataset.loc[ dataset['Age'] <= 16, 'Age']                          = 0
    dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1
    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2
    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3
    dataset.loc[ dataset['Age'] > 64, 'Age'] = 4 ;

# 必要ない特徴を削除
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)
train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
test  = test.drop(drop_elements, axis = 1)
# -

# # Visualisations
# 前処理を終えて初めの３行を見て見ると、全て数値データになっていることが確認できます。

train.head(3)

# # Pearson Correlation Heatmap
#
# 特徴量同士の相関をヒートマップにて確認します。

colormap = plt.cm.RdBu
plt.figure(figsize=(14,12))
plt.title('Pearson Correlation of Features', y=1.05, size=15)
sns.heatmap(train.astype(float).corr(), linewidths=0.1,vmax=1.0,
           square=True, cmap=colormap, linecolor='white', annot=True)

# # Takeaway from the Plots
#
# このプロットから、そこまで特徴量が互いに強く相関していないことが分かります。
# 特徴量が互いに独立であるということは、無駄な特徴がないということであり、学習モデルを構築する上で重要です。
# ここでは、ParchとFamilySizeが比較的相関が高めですが、そのまま残しておきます。

# # Pairplots
#
# 最後に、ある特徴から別の特徴へのデータの分布を見ておきます。

g = sns.pairplot(train[[u'Survived', u'Pclass', u'Sex', u'Age', u'Parch', u'Fare', u'Embarked',
       u'FamilySize', u'Title']], hue='Survived', palette = 'seismic',size=1.2,diag_kind = 'kde',diag_kws=dict(shade=True),plot_kws=dict(s=10) )
g.set(xticklabels=[])

# # Ensembling & Stacking models
# スタッキングアンサンブルモデルを作成します。
#
# Helpers via Python Classes
#
# Pythonで先に学習・予測するためのクラスを定義しておきます。

from sklearn.model_selection import ShuffleSplit

# +
# from sklearn.cross_validation import ShuffleSplit

# +
# パラメータ
ntrain = train.shape[0] # 891
ntest = test.shape[0] # 418
SEED = 0
NFOLDS = 5 # 5分割
# kf = KFold(ntrain, n_splits= NFOLDS, random_state=SEED)
kf = KFold(n_splits=NFOLDS,random_state=SEED)
#追記
# kf.get_n_splits(ntrain)

# Sclearn分類機を拡張
class SklearnHelper(object):
    def __init__(self, clf, seed=0, params=None):
        params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)

    def fit(self,x,y):
        return self.clf.fit(x,y)

    def feature_importances(self,x,y):
        print(self.clf.fit(x,y).feature_importances_)

# -

# Out-of-Fold Predictions
#
# スタッキングでは、第二モデルの学習データに、第一のベースモデルの予測が使用されます。
# しかし、全ての学習・テストデータを一度に使ってしまうと、ベースモデルが既にテストデータを見た状態にあるため、第二モデルでオーバーフィットするリスクがあります。
# そのため、交差検証を施します。

def get_oof(clf, x_train, y_train, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

#     for i, (train_index, test_index) in enumerate(kf): #NFOLDS回まわる
    for i, (train_index, test_index) in enumerate(kf.split(ntrain)): #NFOLDS回まわる
        x_tr = x_train[train_index]
        y_tr = y_train[train_index]
        x_te = x_train[test_index]

        clf.train(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

# Generating our Base First-Level Models
# 第一のベースモデルとして、５つのモデルを準備します。
#
# 1. Random Forest classifier
# 2. Extra Trees classifier
# 3. AdaBoost classifer
# 4. Gradient Boosting classifer
# 5. Support Vector Machine

# Parameters
#
# パラメータの一部をリストアップします。
#
# - n_jobs：コア数。-1にすると全てのコア。
# - n_estimators：学習モデルの分類木の数。デフォルトは10。
# - max_depth：木の最大深度。あまりに大きすぎるとオーバーフィットする。
# - verbose：学習プロセス中にテキストを出力するか。０なら出力しない。3なら繰り返し出力する。
# その他の詳細は、Sklearnの公式ウェブサイトをご覧ください。

# +
# 各モデルのパラメータ
# Random Forest
rf_params = {
    'n_jobs': -1,
    'n_estimators': 500,
     'warm_start': True,
     #'max_features': 0.2,
    'max_depth': 6,
    'min_samples_leaf': 2,
    'max_features' : 'sqrt',
    'verbose': 0
}

# Extra Trees
et_params = {
    'n_jobs': -1,
    'n_estimators':500,
    #'max_features': 0.5,
    'max_depth': 8,
    'min_samples_leaf': 2,
    'verbose': 0
}

# AdaBoost
ada_params = {
    'n_estimators': 500,
    'learning_rate' : 0.75
}

# Gradient Boosting
gb_params = {
    'n_estimators': 500,
     #'max_features': 0.2,
    'max_depth': 5,
    'min_samples_leaf': 2,
    'verbose': 0
}

# Support Vector Classifier
svc_params = {
    'kernel' : 'linear',
    'C' : 0.025
    }
# -

# さらに、Helpers via Python Classesで作成したクラスを用いて、5つの学習モデルのオブジェクトを作成します。

# 5つの学習モデルのオブジェクトを作成
rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)
et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)
ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)
gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)
svc = SklearnHelper(clf=SVC, seed=SEED, params=svc_params)

# Creating NumPy arrays out of our train and test sets
#
# ベースモデルへの入力用データをNumPy配列で準備します。

# 入力データの作成
y_train = train['Survived'].ravel()
train = train.drop(['Survived'], axis=1)
x_train = train.values # 学習データ
x_test = test.values # テストデータ

# Output of the First level Predictions
#
# 学習データとテストデータを５つのベースモデルに送り、交差検証(get_oof関数)を行い、予測を行います。
# 以下のプログラムの実行には、数分を要します。

# +
# 5つのベースモデルで予測
et_oof_train, et_oof_test = get_oof(et, x_train, y_train, x_test) # Extra Trees Classifier
rf_oof_train, rf_oof_test = get_oof(rf,x_train, y_train, x_test) # Random Forest Classifier
ada_oof_train, ada_oof_test = get_oof(ada, x_train, y_train, x_test) # AdaBoost Classifier
gb_oof_train, gb_oof_test = get_oof(gb,x_train, y_train, x_test) # Gradient Boost Classifier
svc_oof_train, svc_oof_test = get_oof(svc,x_train, y_train, x_test) # Support Vector Classifier

print("Training is complete")
# -

Pythonでアンサンブル(スタッキング)学習 & 機械学習チュートリアル in Kaggle/スタッキング法でのアンサンブル学習@titanic.py

スタッキング法でのアンサンブル学習@titanic.py


from django.conf.urls import url,static
from . import views

app_name = 'PURCHASES'

urlpatterns = [
    url(r'^$',views.purchase_home,name="purchase"),
    url(r'^Suppliers/$',views.purchase_supplier,name="purchase_supplier"),
    url(r'^Suppliers/(?P<supplier_id>[0-9]+)/Payment/$',views.purchase_supplier_payment,name="purchase_supplier_payment"),
    url(r'^Invoice/$',views.purchase_add_invoice, name="purchase_add_invoice"),
    url(r'^Invoice/(?P<inv_id>[0-9]+)/$',views.purchase_invoice_item,name="purchase_add_item"),
    url(r'^Invoice/(?P<inv_id>[0-9]+)/Delivered/$',views.purchase_invoice_delivered,name="purchase_invoice_delivered"),
    url(r'^Invoice/(?P<inv_id>[0-9]+)/Item/(?P<item_id>[0-9]+)/Delete$',views.purchase_item_delete,name="purchase_item_delete"),
    url(r'^Invoice/(?P<inv_id>[0-9]+)/Print/$',views.purchase_print_invoice,name="purchase_print_invoice"),
    ]


from PyQt5.QtWidgets import QLabel
from PyQt5.QtGui import QPainter
import cv2
from Board.ImgProcess import ImgProcess

class ShowBoard(QLabel, ImgProcess):
    def __init__(self, parent=None):
        super().__init__(parent)

        self.imgLayer = None
        self.imgLoc = (0, 0)

    def saveImg(self, fpath):
        img = self.Qimg2opencv(self.imgLayer)
        cv2.imwrite(fpath, img)

    def revealImg(self):
        # if self.imgLayer.width() > self.width() or self.imgLayer.height() > self.height():
        scale_x = (self.width() - 80) / self.imgLayer.width()
        scale_y = (self.height() - 80) / self.imgLayer.height()
        scale = min(scale_x, scale_y)

        size = self.imgLayer.size()
        self.imgLayer = self.imgLayer.scaled(scale * size)

        # 图片居中，记录左上角坐标
        x = int((self.width() - self.imgLayer.width()) / 2)
        y = int((self.height() - self.imgLayer.height()) / 2)
        self.imgLoc = (x, y)

        painter = QPainter(self)
        painter.begin(self)
        x, y = self.imgLoc[:]
        painter.drawPixmap(x, y, self.imgLayer)
        painter.end()

    def waiting(self):
        """等待AI上色时展板模糊效果"""
        """暂未完成的功能(3)——不知道为啥不起作用，只能先注释掉了"""
        pass
        # img = self.Qimg2opencv(self.imgLayer)
        #
        # height, width = img.shape[:2]
        # k_size = int(min(height, width) / 2)
        # img = cv2.GaussianBlur(img, ksize=(k_size, k_size), sigmaX=15, sigmaY=15)
        #
        # self.imgLayer = self.opencv2Qimg(img)
        # self.update()

    def paintEvent(self, QPaintEvent):
        super().paintEvent(QPaintEvent)

        if self.imgLayer != None:
            self.revealImg()

    def loadImg(self, img):
        pixmap = self.opencv2Qimg(img)

        self.imgLayer = pixmap
        self.update()


import logging
import os
import random
import sys
import time

import requests
from requests.exceptions import ChunkedEncodingError
from requests.exceptions import ConnectionError
from requests.exceptions import ReadTimeout

from http_request_randomizer.requests.errors.ProxyListException import ProxyListException
from http_request_randomizer.requests.parsers.FreeProxyParser import FreeProxyParser
from http_request_randomizer.requests.parsers.ProxyDbParser import ProxyDbParser
from http_request_randomizer.requests.parsers.ProxyForEuParser import ProxyForEuParser
from http_request_randomizer.requests.parsers.RebroWeeblyParser import RebroWeeblyParser
from http_request_randomizer.requests.parsers.SamairProxyParser import SamairProxyParser
from http_request_randomizer.requests.useragent.userAgent import UserAgentManager

__author__ = 'pgaref'
sys.path.insert(0, os.path.abspath('../../../../'))

# Push back requests library to at least warnings
logging.getLogger("requests").setLevel(logging.WARNING)
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(name)-6s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)

class RequestProxy:
    def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
        self.userAgent = UserAgentManager()
        self.logger = logging.getLogger()
        self.logger.addHandler(handler)
        self.logger.setLevel(0)

        #####
        # Each of the classes below implements a specific URL Parser
        #####
        parsers = list([])
        parsers.append(FreeProxyParser('http://free-proxy-list.net', timeout=timeout))
        parsers.append(ProxyDbParser('http://proxydb.net/?protocol=http&anonlvl=1&anonlvl=2', timeout=timeout))
        parsers.append(ProxyForEuParser('http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
        parsers.append(RebroWeeblyParser('http://rebro.weebly.com', timeout=timeout))
        parsers.append(SamairProxyParser('http://samair.ru/proxy/time-01.htm', timeout=timeout))

        self.logger.debug("=== Initialized Proxy Parsers ===")
        for i in range(len(parsers)):
            self.logger.debug("\t {0}".format(parsers[i].__str__()))
        self.logger.debug("=================================")

        self.sustain = sustain
        self.parsers = parsers
        self.proxy_list = web_proxy_list
        for i in range(len(parsers)):
            try:
                self.proxy_list += parsers[i].parse_proxyList()
            except ReadTimeout:
                self.logger.warn("Proxy Parser: '{}' TimedOut!".format(parsers[i].url))
        self.current_proxy = self.randomize_proxy()

    def set_logger_level(self, level):
        self.logger.setLevel(level)

    def get_proxy_list(self):
        return self.proxy_list

    def generate_random_request_headers(self):
        headers = {
            "Connection": "close",  # another way to cover tracks
            "User-Agent": self.userAgent.get_random_user_agent()
        }  # select a random user agent
        return headers

    def randomize_proxy(self):
        if len(self.proxy_list) == 0:
            raise ProxyListException("list is empty")
        rand_proxy = random.choice(self.proxy_list)
        while not rand_proxy:
            rand_proxy = random.choice(self.proxy_list)
        self.current_proxy = rand_proxy
        return rand_proxy

    #####
    # Proxy format:
    # http://<USERNAME>:<PASSWORD>@<IP-ADDR>:<PORT>
    #####
    def generate_proxied_request(self, url, method="GET", params={}, data={}, headers={}, req_timeout=30):
        try:
            random.shuffle(self.proxy_list)
            # req_headers = dict(params.items() + self.generate_random_request_headers().items())

            req_headers = dict(params.items())
            req_headers_random = dict(self.generate_random_request_headers().items())
            req_headers.update(req_headers_random)

            if not self.sustain:
                self.randomize_proxy()

            headers.update(req_headers)

            self.logger.debug("Using proxy: {0}".format(str(self.current_proxy)))
            request = requests.request(method, url, proxies={"http": self.current_proxy},
                                       headers=headers, data=data, params=params, timeout=req_timeout)
            # Avoid HTTP request errors
            if request.status_code == 409:
                raise ConnectionError("HTTP Response [409] - Possible Cloudflare DNS resolution error")
            elif request.status_code == 403:
                raise ConnectionError("HTTP Response [403] - Permission denied error")
            elif request.status_code == 503:
                raise ConnectionError("HTTP Response [503] - Service unavailable error")
            print('RR Status {}'.format(request.status_code))
            return request
        except ConnectionError:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Proxy unreachable - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except ReadTimeout:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Read timed out - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except ChunkedEncodingError:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Wrong server chunked encoding - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()

if __name__ == '__main__':

    start = time.time()
    req_proxy = RequestProxy()
    print("Initialization took: {0} sec".format((time.time() - start)))
    print("Size: {0}".format(len(req_proxy.get_proxy_list())))
    print("ALL = {0} ".format(req_proxy.get_proxy_list()))

    test_url = 'http://ipv4.icanhazip.com'

    while True:
        start = time.time()
        request = req_proxy.generate_proxied_request(test_url)
        print("Proxied Request Took: {0} sec => Status: {1}".format((time.time() - start), request.__str__()))
        if request is not None:
            print("\t Response: ip={0}".format(u''.join(request.text).encode('utf-8')))
        print("Proxy List Size: {0}".format(len(req_proxy.get_proxy_list())))

        print("-> Going to sleep..")
        time.sleep(10)


import utils.utils as ut

def remove_duplicates(input_files, output_file, config):
    """
    Remove duplicates from bam file
    """
    rmdup_cmd = config['software']['paths']['SAMTOOLSPATH'] + " rmdup " + input_files +\
                " " + output_file
    print(rmdup_cmd)
    ut.run_cmd(rmdup_cmd, 1)

    index_cmd = config['software']['paths']['SAMTOOLSPATH'] + ' index ' + output_file
    ut.run_cmd(index_cmd, 1)


import tornado.ioloop
import tornado.web
import os

#from tornado.options import define, options
#define("port", default=8003, help="run on the given port", type=int)

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        movie = dict()
        movie['folder'] = self.get_argument('film', False)

        if movie['folder'] == False:
            self.render('movie/list.html')
            return

        movie['path'] = os.path.join(os.path.dirname(__file__), os.path.pardir, 'static', 'movie' ,'moviefiles', movie['folder'], )

        fp = open(movie['path'] + '/info.txt')
        movie['name'] = fp.readline().strip('\n')
        movie['year'] = fp.readline().strip('\n')
        movie['rate'] = fp.readline().strip('\n')
        movie['rate'] = int(movie['rate'])
        movie['reviews'] = fp.readline().strip('\n')
        movie['reviews'] = int(movie['reviews'])
        fp.close()

        overviews = dict()
        fp = open(movie['path'] + '/generaloverview.txt')
        overviews = fp.readlines()
        fp.close()

        reviewsFiles = os.listdir(movie['path'])
        count = 0
        for i in range(len(reviewsFiles)):
            if reviewsFiles[i][:6] != 'review':
                reviewsFiles[i] = False
                count = count + 1
        for i in range(count):
            reviewsFiles.remove(False)

        reviews = []
        count = 0
        for file in reviewsFiles:
            fp = open(movie['path'] + '/' + file)
            reviews.append(dict())
            reviews[count]['comment'] = fp.readline().strip('\n')
            reviews[count]['rate'] = fp.readline().strip('\n')
            reviews[count]['name'] = fp.readline().strip('\n')
            reviews[count]['organization'] = fp.readline().strip('\n')
            count = count + 1
            fp.close()

        reviewsL = []
        for i in range(len(reviews)/2):
            reviewsL.append(reviews.pop(0))
        reviewsR = reviews

        self.render('movie/movie.html', movie = movie, overviews = overviews,
                    reviewsL = reviewsL, reviewsR = reviewsR)

application = tornado.web.Application(
    [
    (r"/", MainHandler),
    ],
    static_path = os.path.join(os.path.dirname(__file__), "static"),
    template_path=os.path.join(os.path.dirname(__file__), "templates"),
    debug = True)

if __name__ == "__main__":
    tornado.options.parse_command_line()
    application.listen(options.port)
    tornado.ioloop.IOLoop.instance().start()


import torch
import torch.nn as nn
import torch.nn.functional as F

class Translate(nn.Module):

    def __init__(self,encoder,decoder):
        super(Translate,self).__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, sequence):
        return self.decode(self.encode(sequence))

    def encode(self,sequence):
        return self.encoder(sequence)

    def decode(self,code):
        return self.decoder(code)

class Encoder(nn.Module):

    def __init__(self,n):
        super(Encoder,self).__init__()
        self.n = n
        self.fc1 = nn.Linear(2*self.n,100)
        self.fc2 = nn.Linear(100,100)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(p=0.6)

    def forward(self,x):
        x = x.view(-1,2*self.n)
        x = self.dropout(x)
        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        return x

class Decoder(nn.Module):

    def __init__(self, n, temperature=1):
        super(Decoder,self).__init__()
        self.temperature = temperature
        self.n = n
        self.fc1 = nn.Linear(100,100)
        self.fc2 = nn.Linear(100,3*(self.n-1))
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(p=0.6)

    def forward(self,x):
        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        x = x.view(-1,self.n-1,3)
        x /= self.temperature
        x = F.softmax(x,dim=2)
        return x