from PyQt5.uic import loadUi
from enum import unique
from operator import index
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import Qt, QAbstractTableModel
from PyQt5.QtGui import QPixmap, QImage, QFont
from PyQt5.QtWidgets import QFileDialog , QTableWidget,QMessageBox, QAction, QMainWindow, QTableView, QPushButton, QToolTip, QApplication, QTableWidgetItem

import os
import nltk # Library nltk
# nltk.download('stopwords')
import string # Library string
import re # Library regex
import pandas as pd # Library Pandax

#boolean
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer
import glob
import numpy as np
import sys
import math
Stopwords = set(stopwords.words('english'))

#indexing data
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# ======== Tokenize ========
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.tokenize import word_tokenize

# ======== StopWord & Stemmer ========
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from Sastrawi.Dictionary.ArrayDictionary import ArrayDictionary
from Sastrawi.StopWordRemover.StopWordRemover import StopWordRemover
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory

# ========= Boolean ===========
from contextlib import redirect_stdout

class Ui_MainWindow(QMainWindow):

    def __init__(self):
        QMainWindow.__init__(self)
        loadUi('stkifix.ui', self)

        #Deklarasi
        self.savefile = []
        self.filename = []
        self.allWord = []
        self.token_word = []

        self.stopWord_word = []
        self.stemming_word = []

        #Function Button
        self.pushButton.clicked.connect(self.addFile)
        self.pushButton_2.clicked.connect(self.resetFile)
        self.pushButton_3.clicked.connect(self.booleanSearch)
        self.pushButton_7.clicked.connect(self.preprocessingFile)
        self.pushButton_4.clicked.connect(self.tf_idf)
        self.pushButton_5.clicked.connect(self.jaccardFunction)
        self.pushButton_6.clicked.connect(self.ngramFunction)
        self.pushButton_8.clicked.connect(self.cosineSimilarity)

    def addFile(self):
        self.fileName, fileType = QFileDialog.getOpenFileName(self.centralwidget, "Open File", "",
                                                              "*.txt;;All Files(*)")
        self.filePathName = os.path.basename(self.fileName)
        self.split = os.path.splitext(self.filePathName)[0]
        self.savefile.append(self.fileName)
        self.filename.append(self.split)
        self.listWidget.addItem(self.fileName)

        self.casefoldingWord(self.fileName)

        self.processingFile()
        self.token_word.extend(self.jadi)
        self.stopWord_word.extend(self.tokeStop)

        self.stemming_word.extend(self.tokenStem)
        self.stemming_word = list(dict.fromkeys(self.stemming_word))

        self.invertedIndex()
        self.printIncidence()

    def resetFile(self):
        self.stopWord_word.clear()
        self.stemming_word.clear()
        self.items_clear()

        self.tableWidget.clear()
        self.listWidget.clear()
        self.listWidget_2.clear()
        self.listWidget_3.clear()
        self.listWidget_4.clear()
        self.savefile.clear()

    def getFileNameOnly(self,filenameonly):
        filePathName = os.path.basename(filenameonly)
        splitName = os.path.splitext(filePathName)[0]
        return splitName

    def casefoldingWord(self,fileName):

        self.kalimatOpen = open(fileName,'r').read()
        self.tokenize = self.kalimatOpen.lower()
        # Menghapus angka
        self.angka = re.sub(r"\d+", "", self.tokenize)
        # Menghapus tanda baca
        self.tandabaca = self.angka.translate(str.maketrans("","",string.punctuation))
        # Menghapus whitespace
        self.whitespace = self.tandabaca.strip()
        # Menghapus beberapa whitespace menjadi whitespace tunggal
        self.whitespacetunggal = re.sub('\s+',' ',self.whitespace)

    def processingFile(self):
        # Case Folding -> Tokenizing -> Filtering (Stopword) -> Stemming
        # Tokenizing
        self.jadi = nltk.tokenize.word_tokenize(self.whitespacetunggal)
        self.frekuensi_tokens = nltk.FreqDist(self.jadi).most_common()

        #Filtering (StopWord)
        factory = StopWordRemoverFactory()
        stopword = factory.create_stop_word_remover()
        self.stop = stopword.remove(self.whitespacetunggal)
        self.tokeStop = nltk.tokenize.word_tokenize(self.stop)

        # Stemming
        factory = StemmerFactory()
        self.stemmer = factory.create_stemmer()
        self.stemming   = self.stemmer.stem(self.stop)
        self.tokenStem = nltk.tokenize.word_tokenize(self.stemming)
        # self.kalimatOpen = nltk.tokenize.word_tokenize(self.stop)

    def preprocessingFile(self):
        self.listWidget_2.clear()
        self.listWidget_3.clear()
        self.listWidget_4.clear()
        self.fileNamePre = self.listWidget.currentItem().text()
        self.splitPre = self.getFileNameOnly(self.fileNamePre)
        self.casefoldingWord(self.fileNamePre)

        self.processingFile()

        #Add listWidget
        self.label_2.setText('Kalimat Text : {}'.format(self.kalimatOpen))
        self.listWidget_2.addItem('Token Kata : \n{}'.format(self.jadi))
        self.listWidget_2.addItem('\nFrekuensi Kata : \n{}'.format(self.frekuensi_tokens))
        self.listWidget_3.addItem('Stopword : \n{}'.format(self.stop))
        self.listWidget_4.addItem('Stemming : \n{}'.format(self.stemming))

        # self.counter += 1
        # self.increase += 1

    def invertedIndex(self):
        self.listWidget_7.clear()

        for item in range(len(self.stemming_word)):
            exist_file = list()
            for file in self.savefile:
                with open(file, 'r') as namaFile:
                    for isi in namaFile:
                        isi = isi.lower()
                        if self.stemming_word[item] in isi:
                            exist_file.append(os.path.basename(file))
                            exist_file = list(dict.fromkeys(exist_file))

            self.listWidget_7.addItem('{}\t: <{}>'.format(self.stemming_word[item], exist_file))

    def printIncidence(self):

        self.items_clear()

        panjang_col = len(self.savefile)
        panjang_row = len(self.stemming_word)

        self.tableWidget_2.setColumnCount(panjang_col)
        self.tableWidget_2.setRowCount(panjang_row)

        self.tableWidget_2.setHorizontalHeaderLabels(self.filename)
        self.tableWidget_2.setVerticalHeaderLabels(self.stemming_word)

        for x in range(len(self.stemming_word)):
            for y in range(len(self.filename)):
                with open(self.savefile[y], 'r') as openFile:
                    for content in openFile:
                        if self.stemming_word[x] in content.lower():
                            self.tableWidget_2.setItem(x, y, QTableWidgetItem('1'))
                        else:
                            self.tableWidget_2.setItem(x, y, QTableWidgetItem('0'))

    # =========== Boolean ==============
    def finding_all_unique_words_and_freq(self,words):
        words_unique = []
        word_freq = {}
        for word in words:
            if word not in words_unique:
                words_unique.append(word)
        for word in words_unique:
            word_freq[word] = words.count(word)
        return word_freq

    def finding_freq_of_word_in_doc(self,word,words):
        freq = words.count(word)

    def remove_special_characters(self,text):
        regex = re.compile('[^a-zA-Z0-9\s]')
        text_returned = re.sub(regex,'',text)
        return text_returned

    def booleanSearch(self):

        self.listWidget_12.clear()
        self.listWidget_5.clear()
        totalBoolean = []
        for count in self.savefile:
            totalBoolean.append(count)

        all_words = []
        dict_global = {}
        idx = 1
        files_with_index = {}
        for file in self.savefile:
            fname = file
            file = open(file , "r")
            text = file.read()
            text = self.remove_special_characters(text)
            text = re.sub(re.compile('\d'),'',text)
            sentences = sent_tokenize(text)
            words = word_tokenize(text)
            words = [word for word in words if len(words)>1]
            words = [word.lower() for word in words]
            words = [word for word in words if word not in Stopwords]
            dict_global.update(self.finding_all_unique_words_and_freq(words))
            files_with_index[idx] = os.path.basename(fname)
            idx = idx + 1

        unique_words_all = set(dict_global.keys())

        linked_list_data = {}
        for word in unique_words_all:
            linked_list_data[word] = SlinkedList()
            linked_list_data[word].head = Node(1,Node)
        word_freq_in_doc = {}
        idx = 1
        for file in self.savefile:
            file = open(file, "r")
            text = file.read()
            text = self.remove_special_characters(text)
            text = re.sub(re.compile('\d'),'',text)
            sentences = sent_tokenize(text)
            words = word_tokenize(text)
            words = [word for word in words if len(words)>1]
            words = [word.lower() for word in words]
            words = [word for word in words if word not in Stopwords]
            word_freq_in_doc = self.finding_all_unique_words_and_freq(words)
            for word in word_freq_in_doc.keys():
                linked_list = linked_list_data[word].head
                while linked_list.nextval is not None:
                    linked_list = linked_list.nextval
                linked_list.nextval = Node(idx ,word_freq_in_doc[word])
            idx = idx + 1

            #Step -6 Query processing and output generation
            userInput = self.textEdit.toPlainText().lower()
            query = word_tokenize(userInput)
            connecting_words = []
            cnt = 1
            different_words = []
            for word in query:
                if word.lower() != "and" and word.lower() != "or" and word.lower() != "not":
                    different_words.append(word.lower())
                else:
                    connecting_words.append(word.lower())

            total_files = len(files_with_index)
            zeroes_and_ones = []
            zeroes_and_ones_of_all_words = []
            for word in (different_words):
                if word.lower() in unique_words_all:
                    zeroes_and_ones = [0] * total_files
                    linkedlist = linked_list_data[word].head
                    # self.listWidget_5.addItem(word)
                    while linkedlist.nextval is not None:
                        zeroes_and_ones[linkedlist.nextval.doc - 1] = 1
                        linkedlist = linkedlist.nextval
                    zeroes_and_ones_of_all_words.append(zeroes_and_ones)
                else:
                    self.listWidget_5.addItem("{} not found".format(word))
            # print(zeroes_and_ones_of_all_words)
            # for zeroes_word in zeroes_and_ones_of_all_words:
            #     self.listWidget_5.addItem(str(zeroes_word))
            for word in connecting_words:
                word_list1 = zeroes_and_ones_of_all_words[0]
                word_list2 = zeroes_and_ones_of_all_words[1]
                if word == "and":
                    bitwise_op = [w1 & w2 for (w1,w2) in zip(word_list1,word_list2)]
                    zeroes_and_ones_of_all_words.remove(word_list1)
                    zeroes_and_ones_of_all_words.remove(word_list2)
                    zeroes_and_ones_of_all_words.insert(0, bitwise_op);
                elif word == "or":
                    bitwise_op = [w1 | w2 for (w1,w2) in zip(word_list1,word_list2)]
                    zeroes_and_ones_of_all_words.remove(word_list1)
                    zeroes_and_ones_of_all_words.remove(word_list2)
                    zeroes_and_ones_of_all_words.insert(0, bitwise_op);
                elif word == "not":
                    bitwise_op = [not w1 for w1 in word_list2]
                    bitwise_op = [int(b == True) for b in bitwise_op]
                    zeroes_and_ones_of_all_words.remove(word_list2)
                    zeroes_and_ones_of_all_words.remove(word_list1)
                    bitwise_op = [w1 & w2 for (w1,w2) in zip(word_list1,bitwise_op)]
                    zeroes_and_ones_of_all_words.insert(0, bitwise_op);

            files = []

            print(zeroes_and_ones_of_all_words)

            lis = zeroes_and_ones_of_all_words[0]
            cnt = 1
            for index in lis:
                if index == 1:
                    files.append(files_with_index[cnt])
                cnt = cnt+1

        self.listWidget_12.addItem(str(zeroes_and_ones_of_all_words))
        self.listWidget_5.addItem(str(files))

    # ========= TF/IDF ===========

    def tf_idf(self):

        total = []
        for count in self.savefile:
            total.append(0)

        # ========================= Buat Kolom ===================================
        font = QFont()
        font.setBold(True)

        userInput = self.textEdit_2.toPlainText().lower()
        # ============== PREPROCESSING KEYWORD ======================
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        sentenceIsi = ' '.join(tokenStem)
        # ============== END PREPROCESSING KEYWORD ===================
        userInput = sentenceIsi.split()

        kolom_tf = ['df', 'D/df', 'IDF', 'IDF+1']

        jarak_W = len(self.savefile) + len(kolom_tf)
        panjang_kolom = len(self.savefile)*2 + len(kolom_tf) + 1

        self.tableWidget.setColumnCount(panjang_kolom)
        self.tableWidget.setRowCount(len(userInput)+3)

        self.tableWidget.horizontalHeader().setVisible(False)
        self.tableWidget.verticalHeader().setVisible(False)

        # ========== Span Tf ==========

        ''' Format bikin span : tableTf.setSpan(row, column, rowSpan, columnSpan) '''

        self.tableWidget.setSpan(0, 1, 1, len(self.savefile))
        newItem = QTableWidgetItem("tf")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, 1, newItem)
        self.tableWidget.item(0, 1).setFont(font)

        # ========== Span df ==========

        self.tableWidget.setSpan(0, len(self.savefile)+1, 2, 1)
        newItem = QTableWidgetItem("df")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+1, newItem)
        self.tableWidget.item(0, len(self.savefile)+1).setFont(font)

        # ========== Span D/df ==========

        self.tableWidget.setSpan(0, len(self.savefile)+2, 2, 1)
        newItem = QTableWidgetItem("D / df")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+2, newItem)
        self.tableWidget.item(0, len(self.savefile)+2).setFont(font)

        # ========== Span IDF ==========

        self.tableWidget.setSpan(0, len(self.savefile)+3, 2, 1)
        newItem = QTableWidgetItem("IDF")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+3, newItem)
        self.tableWidget.item(0, len(self.savefile)+3).setFont(font)

        # ========== Span IDF+1 ==========

        self.tableWidget.setSpan(0, len(self.savefile)+4, 2, 1)
        newItem = QTableWidgetItem("IDF + 1")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+4, newItem)
        self.tableWidget.item(0, len(self.savefile)+4).setFont(font)

        # ========== Span W ==========

        self.tableWidget.setSpan(0, len(self.savefile)+5, 1, len(self.savefile))
        newItem = QTableWidgetItem("W = tf*(IDF+1)")
        newItem.setTextAlignment(QtCore.Qt.AlignCenter)
        self.tableWidget.setItem(0, len(self.savefile)+5, newItem)
        self.tableWidget.item(0, len(self.savefile)+5).setFont(font)

        # ========== MAKE TABLE ==========

        # __Print Document di kolom tf (kiri)__
        for y in range(len(self.savefile)):
            cell_item = QTableWidgetItem(str(self.filename[y]))
            cell_item.setTextAlignment(QtCore.Qt.AlignCenter)
            self.tableWidget.setItem(1, y+1, cell_item)
            self.tableWidget.item(1, y+1).setFont(font)

        # __Print Document di kolom tf (kanan)__
        for y in range(len(self.savefile)):
            cell_item = QTableWidgetItem(str(self.filename[y]))
            cell_item.setTextAlignment(QtCore.Qt.AlignCenter)
            self.tableWidget.setItem(1, y+1+jarak_W, cell_item)
            self.tableWidget.item(1, y+1+jarak_W).setFont(font)

        # ====================== END BUAT KOLOM ===============================

        # # __Isi Tf-Idf__
        for x in range(len(userInput)): # 0 < x < 3 (x = 0)

            exist_in = []

            for y in range((panjang_kolom)): # 0 < y < 11 (y = 0)

                # __Print userInput di row__
                self.tableWidget.setItem(x+2, 0, QTableWidgetItem(str(userInput[x])))

                # __Print nilai tf per document__
                if y < len(self.savefile):
                    with open(self.savefile[y], 'r') as openFile:
                        for content in openFile:
                            if userInput[x] in content.lower():
                                exist_in.append(1)
                            else:
                                exist_in.append(0)

                    self.tableWidget.setItem(x+2, y+1, QTableWidgetItem(str(exist_in[y])))

                # __df__
                df = 0
                for count in exist_in:
                    if count > 0:
                        df = df + 1 # df = 1

                # __D/df__
                if df != 0:
                    D = round(len(self.savefile) / df, 3)
                else:
                    D = 1

                # __idf__
                idf = round(math.log(D), 3)

                # __idf+1__
                idf_1 = round(idf+1,3)

                # __W__
                W = []
                for freq in range(len(exist_in)):
                    W.append(round(exist_in[freq]*(idf+1), 3))

                if len(W) == len(self.savefile) and y == len(self.savefile):
                    # __SUM__
                    zipped_list = zip(total, W)
                    total = [x+y for (x, y) in zipped_list]

                # __Print nilai setelah tf dan sebelum W__
                if y == (len(self.savefile)+1) and y <= jarak_W:
                    self.tableWidget.setItem(x+2, y, QTableWidgetItem(str(df)))
                    self.tableWidget.setItem(x+2, y+1, QTableWidgetItem(str(D)))
                    self.tableWidget.setItem(x+2, y+2, QTableWidgetItem(str(idf)))
                    self.tableWidget.setItem(x+2, y+3, QTableWidgetItem(str(idf_1)))

                # __Print nilai W per dokumen__
                if y > jarak_W:
                    self.tableWidget.setItem(x+2, y, QTableWidgetItem(str(W[y-(jarak_W+1)])))

                if x == len(userInput)-1: # x = 2. userInput = 2

                    if y == jarak_W: # y = 7
                        item_sum = QTableWidgetItem('Sum :')
                        self.tableWidget.setItem(x+3, y, item_sum)
                        self.tableWidget.item(x+3, y).setFont(font)
                    elif y > jarak_W:
                        total_sum = QTableWidgetItem(str(total[y-(jarak_W+1)]))
                        self.tableWidget.setItem(x+3, y, total_sum)
                        self.tableWidget.item(x+3, y).setFont(font)

        file_rank = self.filename.copy()

        self.listWidget_8.addItem('Ranking:')

        for i in range(len(self.savefile)-1):
            for j in range(len(self.savefile)-i-1):
                if total[j] < total[j+1]:
                    temp_total = total[j]
                    total[j] = total[j+1]
                    total[j+1] = temp_total

                    temp = file_rank[j]
                    file_rank[j] = file_rank[j+1]
                    file_rank[j+1] = temp

        print(type(file_rank))
        for i in range(len(file_rank)):
            print(i)
            self.listWidget_8.addItem('{}. {}'.format(i,file_rank[i]))

        # print(total)
        # print(file_rank)

    # ============= Jaccard =============
    def jaccardFunction(self):
        self.listWidget_6.clear()
        self.listWidget_11.clear()
        total = []
        for count in self.savefile:
            total.append(count)

        userInput = self.textEdit_3.toPlainText().lower()
        # print(userInput)

        # ============== PREPROCESSING KEYWORD ======================
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        sentenceUser = ' '.join(tokenStem)
        # ============== END PREPROCESSING KEYWORD ===================

        userInput = re.split(r'\W+', sentenceUser)

        a = userInput
        self.listWidget_6.addItem('Orignal Text : {}'.format(a))
        increaseNo = 0
        hasil_urutJaccard = list()
        for folder in total:
            self.listWidget_6.addItem('==========================================')
            with open(folder,'r') as namafile:
                for isi in namafile:
                    isi = isi.lower()
                    # ============== PREPROCESSING KEYWORD ======================
                    factory = StopWordRemoverFactory()
                    factoryStem = StemmerFactory()

                    angka = re.sub(r"\d+", "", isi)
                    tandabaca = angka.translate(str.maketrans("","",string.punctuation))
                    whitespace = tandabaca.strip()
                    whitespacetunggal = re.sub('\s+',' ',whitespace)

                    # Stop Word
                    stopword = factory.create_stop_word_remover()
                    stop = stopword.remove(whitespacetunggal)

                    # Stemming
                    stemmer = factoryStem.create_stemmer()
                    stemming   = stemmer.stem(stop)
                    tokenStem = nltk.tokenize.word_tokenize(stemming)
                    sentenceIsi = ' '.join(tokenStem)
                    # ============== END PREPROCESSING KEYWORD ===================
                    isi = sentenceIsi.split()

                    self.listWidget_6.addItem('{} :'.format(self.filename[increaseNo]))
                    self.listWidget_6.addItem('Original Text : [{}]'.format(isi))
                    c = set(a).intersection(isi)
                    d = round(float(len(c)) / (len(a)+len(isi)-len(c)),3)
                    self.listWidget_6.addItem('{} / {} : [{}]'.format(len(c),(len(a)+len(isi)-len(c)),d))
                    hasil_urutJaccard.append([self.filename[increaseNo],d])
                    increaseNo +=1

        # print(d)

        hasil_urutJaccard.sort(key=lambda row: (row[1]),reverse=True)

        increaseNo = 0
        self.listWidget_11.addItem('Ranking')
        for folder in total:
            self.listWidget_11.addItem('{} : [{}]'.format(hasil_urutJaccard[increaseNo][0],hasil_urutJaccard[increaseNo][1]))
            increaseNo+=1

    # ================= N Gram ===============
    def ngramFunction(self):
        self.listWidget_9.clear()
        self.listWidget_10.clear()
        total = []
        for count in self.savefile:
            total.append(count)

        userInput = self.textEdit_4.toPlainText().lower()
        # print(userInput)
        # Casefolding
        # ============== PREPROCESSING KEYWORD ======================
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        # ============== END PREPROCESSING KEYWORD ===================

        sentenceUser = ' '.join(tokenStem)
        userInput = re.split(r'\W+', sentenceUser)

        m = 2 # nilai untuk jumlah grams
        nGramSearchList = self.getNGrams(userInput,m)

        print(nGramSearchList)
        self.listWidget_9.addItem('Orignal Text : {}'.format(nGramSearchList))

        increaseNo = 0
        hasil_urutnGram = list()
        for folder in total:
            self.listWidget_9.addItem('==========================================')
            for i in range(len(nGramSearchList)):
                with open(folder,'r') as namafile:
                    a = 0
                    for isi in namafile:
                        isi = isi.lower()
                        # ========== PREPROCESSING ISI ==================

                        angka = re.sub(r"\d+", "", isi)
                        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
                        whitespace = tandabaca.strip()
                        whitespacetunggal = re.sub('\s+',' ',whitespace)

                        # Stop Word
                        stopword = factory.create_stop_word_remover()
                        stop = stopword.remove(whitespacetunggal)

                        # Stemming
                        stemmer = factoryStem.create_stemmer()
                        stemming   = stemmer.stem(stop)
                        tokenStem = nltk.tokenize.word_tokenize(stemming)
                        sentenceIsi = ' '.join(tokenStem)

                        # ========= END PREPROCESSING ISI ==============
                        isi = sentenceIsi.split()
                        fileGetNGrams = self.getNGrams(isi,m)
                        self.listWidget_9.addItem('{} : \n{}'.format(self.filename[increaseNo],fileGetNGrams))

                        # print(len(fileGetNGrams)) // Jumlah panjang listnya
                        for j in range(len(fileGetNGrams)):
                            if nGramSearchList[i] == fileGetNGrams[j]:
                                a +=1
            d = round(float(a)/(len(isi)),3)
            hasil_urutnGram.append([self.filename[increaseNo],d])
            self.listWidget_9.addItem('{} / {}: {}'.format(a,len(isi),d))
            increaseNo +=1

        hasil_urutnGram.sort(key=lambda row: (row[1]),reverse=True)

        increaseNo = 0
        self.listWidget_10.addItem('Ranking')
        for folder in total:
            self.listWidget_10.addItem('{} : [{}]'.format(hasil_urutnGram[increaseNo][0],hasil_urutnGram[increaseNo][1]))
            increaseNo+=1

    def getNGrams(self,wordlist,n):
        ngrams = []
        for i in range(len(wordlist)-(n-1)):
            ngrams.append(wordlist[i:i+n])
        return ngrams

    def items_clear(self):
        for item in self.tableWidget_2.selectedItems():
            newitem = QTableWidgetItem()
            self.tableWidget_2.setItem(item.row(), item.column(), newitem)

    # =================== Cosine Similarity =============
    def cosineSimilarity(self):
        self.listWidget_13.clear()

        header_file = []
        documents = []
        total = []

        for count in self.savefile:
            total.append(count)

        userInput = self.textEdit_5.toPlainText().lower()

        # ======== Preprocessing ==========
        # Casefolding
        factory = StopWordRemoverFactory()
        factoryStem = StemmerFactory()

        angka = re.sub(r"\d+", "", userInput)
        tandabaca = angka.translate(str.maketrans("","",string.punctuation))
        whitespace = tandabaca.strip()
        whitespacetunggal = re.sub('\s+',' ',whitespace)

        # Stop Word
        stopword = factory.create_stop_word_remover()
        stop = stopword.remove(whitespacetunggal)

        # Stemming
        stemmer = factoryStem.create_stemmer()
        stemming   = stemmer.stem(stop)
        tokenStem = nltk.tokenize.word_tokenize(stemming)
        # ======== Stem Keyword End ======

        sentenceUser = ' '.join(tokenStem)
        documents.append(sentenceUser)
        print(documents)

        header_file.append("User Input")
        header_file.extend(self.filename)
        print(header_file)

        for i in range(len(total)):
            document_word = []
            with open(total[i],'r') as file:
                for isi in file:
                    tokenize = isi.lower()
                    angka = re.sub(r"\d+", "", tokenize)
                    tandabaca = angka.translate(str.maketrans("","",string.punctuation))
                    whitespace = tandabaca.strip()
                    whitespacetunggal = re.sub('\s+',' ',whitespace)

                    stopword = factory.create_stop_word_remover()
                    stop = stopword.remove(whitespacetunggal)

                    # Stemming
                    stemmer = factoryStem.create_stemmer()
                    stemming   = stemmer.stem(stop)
                    tokenStem = nltk.tokenize.word_tokenize(stemming)

                    document_word.extend(tokenStem)
            sentence = ' '.join(document_word)
            documents.append(sentence)

        print(documents)

        count_vectorizer = CountVectorizer(stop_words=stopword)
        count_vectorizer = CountVectorizer()
        sparse_matrix = count_vectorizer.fit_transform(documents)

        doc_term_matrix = sparse_matrix.todense()
        df = pd.DataFrame(doc_term_matrix,
                        columns=count_vectorizer.get_feature_names_out(),
                        index=[header_file])

        self.tableWidget_3.setColumnCount(len(df.columns))
        self.tableWidget_3.setRowCount(len(df.index))

        self.tableWidget_3.setHorizontalHeaderLabels(df.columns)
        self.tableWidget_3.setVerticalHeaderLabels(header_file)

        for i in range(len(df.index)):
            for j in range(len(df.columns)):
                self.tableWidget_3.setItem(i,j,QTableWidgetItem(str(df.iloc[i,j])))

        hasil = cosine_similarity(df, df)

        # Cosine_similarity fungsi dari sklearn
        # K(x,y) = <X,y> / (||X||*||Y||)
        print(hasil)

        hasil_urutConsaint = list()
        jmlh_file = 1
        for i in range(len(header_file)-1):
            self.listWidget_13.addItem("{} : {}".format(header_file[jmlh_file],str(round(hasil[0][jmlh_file],4))))
            hasil_urutConsaint.append([header_file[jmlh_file],str(round(hasil[0][jmlh_file],4))])
            jmlh_file +=1

        self.listWidget_13.addItem('=================')

        # print(d)

        hasil_urutConsaint.sort(key=lambda row: (row[1]),reverse=True)

        increaseNo = 0
        self.listWidget_13.addItem('Ranking')
        for folder in total:
            self.listWidget_13.addItem('{} : {}'.format(hasil_urutConsaint[increaseNo][0],hasil_urutConsaint[increaseNo][1]))
            increaseNo+=1

        total.clear()
        header_file.clear()

class Node:
    def __init__(self ,docId, freq = None):
        self.freq = freq
        self.doc = docId
        self.nextval = None

class SlinkedList:
    def __init__(self ,head = None):
        self.head = head

app = QApplication([])
window = Ui_MainWindow()
window.show()
app.exec_()


from django.conf.urls import url

from App import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^(\d+)/$', views.index, name='index1'),
    url(r'^post/$',views.manage_post,name='post'),
    url(r'^add/(\w+)/$',views.add_post,name='add'),
    url(r'^one/$',views.one2one,name='one'),

]


#! /usr/bin/env python3

import sys

filename = sys.argv[1]

from collections import Counter

count = 0

hexamers_5 = Counter()
hexamers_3 = Counter()

for line in open(filename):
    line = line.rstrip()

    if count == 0:
       count += 1

    elif count == 1:
       seq = line
       count += 1

    elif count == 2:
       count += 1

    elif count == 3:
       count = 0

       hexamer_5 = seq[0:6]
       hexamer_3 = seq[-6:]
       hexamers_5[hexamer_5] += 1
       hexamers_3[hexamer_3] += 1

for hexamer, count in hexamers_5.most_common(1):
    print("The most common hexamer at 5':", hexamer, count)

for hexamer, count in hexamers_3.most_common(1):
    print("The most common hexamer at 3':", hexamer, count)


#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import argparse
import json
from collections import OrderedDict

import sys

import boto
import boto.ec2
import os
import providers

def main():
    is_cgi = providers.is_cgi()
    working_dir = providers.get_opscode_root()
    scripts_dir = os.path.join(working_dir, 'operation_code', 'toolchain', 'rundeck', 'scripts')
    sys.path.insert(0, scripts_dir)
    import oobe
    import aws
    import utilities

    if is_cgi:
        args, form = providers.parse_default_cgi_vars()
        args['prefix'] = form['prefix'].value
        args['suffix'] = form['suffix'].value
    else:
        parser = argparse.ArgumentParser(description='Get AMIs.')
        providers.setup_default_args(parser)
        parser.add_argument('--prefix', help="Prefix of AMI Names.", default='OOBE_AMAZON_')
        parser.add_argument('--suffix', help="Suffix of AMI Names.", default=r'\d+\.\d+\.\d+_\d+\.\d+\.\d+')
        args = oobe.parse_args(parser)

    oobe.setup_logging(args['app_log_level'], args['other_log_level'], lambda: providers.rundeck_log_file_name(args['project'], args['environment']))
    logger = utilities.get_logger(__name__)
    temp_dir = None
    try:
        logger.info("Using [%s] and [%s]" % (args['prefix'], args['suffix']))
        deployment_config, temp_dir = oobe.setup_deployment_vars(args, working_dir)

        ec2 = boto.ec2.connect_to_region(args['region'])
        values = [];
        for image in aws.list_images(ec2, args['prefix'], args['suffix']):
            values.append({"name": image.name, "value": image.id})
        print("Content-Type: application/json")  # HTML is following
        print("\n")  # blank line, end of headers
        print(json.dumps(values))
    finally:
        oobe.cleanup(temp_dir)

if __name__ == '__main__':
    main()


import pandas as pd

from flask import (
    Flask,
    render_template,
    jsonify)

from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

# The database URI
app.config['SQLALCHEMY_DATABASE_URI'] = "sqlite:///wine_review.sqlite"

db = SQLAlchemy(app)

# Create database tables
@app.before_first_request
def setup():
    # Recreate database each time for demo
    #db.drop_all()
    db.create_all()

@app.route("/")
def home():

    # Find data
    review_data = SQLAlchemy.db.wine_review.find_one()
    print(review_data)
    # return template and data
    return render_template("index.html", review_data = review_data)

if __name__ == '__main__':
    app.run(debug=True)


import adbase as ad
import json
import datetime

class HomePresenceApp(ad.ADBase):

    def initialize(self):
        self.adbase = self.get_ad_api()
        self.hass = self.get_plugin_api("HASS") #get hass api
        self.mqtt = self.get_plugin_api("MQTT") #get mqtt api

        self.presence_topic = self.args.get("monitor_topic", "presence")
        self.timeout = self.args.get("not_home_timeout", 30) #time interval before declaring not home
        self.minimum_conf = self.args.get("minimum_confidence", 90)
        self.depart_check_time = self.args.get("depart_check_time", 30)
        self.system_timeout = self.args.get("system_timeout", 60) #time interval for system to report back from echo
        system_check = self.args.get("system_check", 30) #time in seconds app queries for system online
        self.all_users_sensors = [] #used to determine if anyone at home or not
        self.not_home_timers = {}
        self.location_timers = {}
        self.home_state_entities = {} #used to store or map different confidence sensors based on location to devices
        self.system_handle = {}

        self.monitor_entity = "{}.monitor_state".format(self.presence_topic) #used to check if the network monitor is busy
        if not self.mqtt.entity_exists(self.monitor_entity):
            self.mqtt.set_state(self.monitor_entity, state = "idle", attributes = {"locations": []}, replace = True) #set it to idle initially

        self.mqtt.listen_state(self.monitor_scan_now, self.monitor_entity, new = "scan")

        self.monitor_handlers = {} #used to store different handlers
        self.monitor_handlers[self.monitor_entity] = None

        self.everyone_not_home = "binary_sensor.everyone_not_home"
        self.everyone_home = "binary_sensor.everyone_home"
        self.somebody_is_home = "binary_sensor.somebody_is_home"

        self.setup_global_sensors() #setup the gbove sensors

        self.gateway_timer = None #run only a single timer at a time, to avoid sending multiple messages to the monitor
        self.motion_timer = None #run only a single timer at a time, to avoid sending multiple messages to the monitor

        """setup home gateway sensors"""
        for gateway_sensor in self.args.get("home_gateway_sensors", []):
            """it is assumed when the sensor is "on" it is opened and "off" is closed"""
            self.hass.listen_state(self.gateway_opened, gateway_sensor) #when the door is either opened or closed

        """setup home motion sensors, used for RSSI"""
        for motion_sensor in self.args.get("home_motion_sensors", []):
            """it is assumed when the sensor is "on" motion detected and "off" motion not detected after timeout"""
            self.hass.listen_state(self.motion_detected, motion_sensor) #if any motion is detected

        time = "00:00:01"
        self.adbase.run_daily(self.restart_device, time, constrain_days = "sun") #restart device at midnight on sunday

        if self.system_timeout > system_check:
            time = datetime.datetime.now() + datetime.timedelta(seconds = 1)
            topic = "{}/echo".format(self.presence_topic)
            self.adbase.run_every(self.send_mqtt_message, time, system_check, topic=topic, payload="", scan_type="System")

        else:
            self.adbase.log("Cannot setup System Check due to System Timeout being Lower than System Check in Seconds", level = "WARNING")

        self.mqtt.listen_event(self.presence_message, "MQTT", wildcard = "{}/#".format(self.presence_topic))
        self.hass.listen_event(self.hass_restarted, "plugin_restarted")
        self.adbase.run_in(self.reload_device_state, 5) #reload systems
        self.adbase.run_in(self.load_known_devices, 0) #load up devices for all locations

    def presence_message(self, event_name, data, kwargs):
        topic = data["topic"]
        payload = data["payload"]
        self.adbase.log("{} payload: {}".format(topic, payload), level = "DEBUG")

        if topic.split("/")[-1] == "status": #meaning its a message on the presence system
            location = topic.split("/")[1].replace("_"," ").title()
            siteId = location.lower().replace(" ", "_")

            self.adbase.log("The Presence System in the {} is {}".format(location, payload.title()), level = "DEBUG")

            if payload.title() == "Offline": #run timer so to clear all entities for that location
                if location in self.location_timers:
                    self.adbase.cancel_timer(self.location_timers[location])

                self.location_timers[location] = self.adbase.run_in(self.clear_location_entities, self.system_timeout, location = location)

            elif payload.title() == "Online" and location in self.location_timers:
                self.adbase.cancel_timer(self.location_timers[location])

            entity_id = "{}.{}".format(self.presence_topic, siteId)
            attributes = {}

            if not self.mqtt.entity_exists(entity_id):
                attributes.update({"friendly_name" : location})
                self.adbase.run_in(self.load_known_devices, 30) #load up devices for all locations

            self.mqtt.set_state(entity_id, state = payload.title(), attributes = attributes)

            if self.system_handle.get(entity_id, None) == None:
                self.system_handle[entity_id] = self.mqtt.listen_state(self.system_state_changed, entity_id, old = "Offline", new = "Online")

            return

        elif topic.split("/")[-1] == "restart": #meaning its a message is a restart
            self.adbase.log("The Presence System is Restarting")
            return

        elif topic.split("/")[-1].lower() in ["depart", "arrive", "known device states", "add static device", "delete static device"]: #meaning its something we not interested in
            return

        try:
            if topic.split("/")[-1] == "rssi": #meaning its for rssi
                attributes = {"rssi" : payload}
            else:
                payload = json.loads(payload)
        except:

            return

        if topic.split("/")[-1] == "start": #meaning a scan is starting
            location = payload["identity"]
            #self.adbase.log("The system in the {} is scanning".format(location))
            if self.mqtt.get_state(self.monitor_entity, copy=False) != "scanning":
                """since its idle, just set it to scanning and put in the location of the scan"""
                self.mqtt.set_state(self.monitor_entity, state = "scanning", attributes = {"scan_type" : topic.split("/")[2], "locations": [location], location : "scanning"})
            else: #meaning it was already set to "scanning" already, so just update the location
                locations_attr = self.mqtt.get_state(self.monitor_entity, attribute = "locations")
                if location not in locations_attr: #meaning it hadn't started the scan before
                    locations_attr.append(location)
                    self.mqtt.set_state(self.monitor_entity, attributes = {"locations": locations_attr, location : "scanning"}) #update the location in the event of different scan systems in place

            return

        elif topic.split("/")[-1] == "end": #meaning a scan in a location just ended
            location = payload["identity"]
            locations_attr = self.mqtt.get_state(self.monitor_entity, attribute = "locations")
            if location in locations_attr: #meaning it had started the scan before
                locations_attr.remove(location)

                if locations_attr == []: #meaning no more locations scanning
                    self.mqtt.set_state(self.monitor_entity, state = "idle", attributes = {"scan_type" : topic.split("/")[2], "locations": [], location : "idle"}) #set the monitor state to idle
                else:
                    self.mqtt.set_state(self.monitor_entity, attributes = {"locations": locations_attr, location : "idle"}) #update the location in the event of different scan systems in place
            return

        elif topic.split("/")[-1] == "echo": #meaning it is for echo check
            self.adbase.log(payload, level = "DEBUG")

            if payload == "ok":
                location = topic.split("/")[1]
                siteId = location.replace(" ", "_").lower()
                entity_id = "{}.{}".format(self.presence_topic, siteId)
                if location in self.location_timers:
                    self.adbase.cancel_timer(self.location_timers[location])

                self.location_timers[location] = self.adbase.run_in(self.clear_location_entities, self.system_timeout, location = location)

                if self.mqtt.get_state(entity_id, copy=False) == "Offline":
                    self.mqtt.set_state(entity_id, state = "Online")
            return

        location = topic.split("/")[1].replace("_"," ").title()
        siteId = location.replace(" ", "_").lower()
        device_name = topic.split("/")[2]
        device_local = "{}_{}".format(device_name, siteId)
        appdaemon_entity = "{}.{}".format(self.presence_topic, device_local)

        if topic.split("/")[-1] == "rssi": #meaning its for rssi

            if topic.split("/")[-2] != "scan": #meaning its not for rssi scan
                self.mqtt.set_state(appdaemon_entity, attributes = attributes)

        elif isinstance(payload, dict) and payload.get("type", None) in ["KNOWN_MAC", "GENERIC_BEACON"]:
            friendly_name = payload.get("name", None)

            if friendly_name != None:
                del payload["name"]
                payload["friendly_name"] = "{} {}".format(friendly_name, location)

            confidence = int(float(payload["confidence"]))
            del payload["confidence"]

            conf_sensor = "sensor.{}".format(device_local)
            device_state = "{}_home".format(device_name)
            user_device_sensor = "binary_sensor.{}".format(device_state)

            if confidence >= self.minimum_conf:
                state = "on"
            else:
                state = "off"

            if not self.hass.entity_exists(conf_sensor): #meaning it doesn't exist
                self.adbase.log("Creating sensor {!r} for Confidence".format(conf_sensor))
                self.hass.set_state(conf_sensor, state = confidence, attributes = {"friendly_name" : "{} {} Confidence".format(friendly_name, location)}) #create sensor for confidence

                """create user home state sensor"""
                if not self.hass.entity_exists(user_device_sensor): #meaning it doesn't exist.
                    self.adbase.log("Creating sensor {!r} for Home State".format(user_device_sensor))

                    self.hass.set_state(user_device_sensor, state = state, attributes = {"friendly_name" : "{} Home".format(friendly_name), "device_class" : "presence"}) #create device sensor

                    if state == "on" and self.hass.get_state(self.somebody_is_home, copy=False) == "off": #at least someone is home
                        self.update_hass_sensor(self.somebody_is_home, "on")

                self.hass.listen_state(self.confidence_updated, conf_sensor, device_state = device_state, immediate = True) #process the change immedaitely

                if device_state not in self.home_state_entities:
                    self.home_state_entities[device_state] = list()

                if conf_sensor not in self.home_state_entities[device_state]: #not really needed, but noting wrong in being extra careful
                    self.home_state_entities[device_state].append(conf_sensor)

            else:
                if device_state not in self.home_state_entities:
                    self.home_state_entities[device_state] = list()

                if conf_sensor not in self.home_state_entities[device_state]:
                    self.home_state_entities[device_state].append(conf_sensor)
                    self.hass.listen_state(self.confidence_updated, conf_sensor, device_state = device_state, immediate = True)

                self.update_hass_sensor(conf_sensor, confidence)

            #add location to payload, so its available in the AD entity's data
            payload["location"] = location #good when writing app to triagulate location
            self.mqtt.set_state(appdaemon_entity, state = confidence, attributes = payload)

            if user_device_sensor not in self.all_users_sensors:
                self.all_users_sensors.append(user_device_sensor)

            if device_state not in self.not_home_timers:
                self.not_home_timers[device_state] = None

    def confidence_updated(self, entity, attribute, old, new, kwargs):
        device_state = kwargs["device_state"]
        user_device_sensor = "binary_sensor." + device_state
        user_conf_sensors = self.home_state_entities.get(device_state, None)

        if user_conf_sensors != None:
            sensor_res = list(map(lambda x: self.hass.get_state(x, copy=False), user_conf_sensors))
            sensor_res = [i for i in sensor_res if i != "unknown"] # remove unknown vales from list
            sensor_res = [i for i in sensor_res if i != None] # remove None values from list
            if  sensor_res != [] and any(list(map(lambda x: int(x) >= self.minimum_conf, sensor_res))): #meaning at least one of them states is greater than the minimum so device definitely home
                if self.not_home_timers[device_state] != None: #cancel timer if running
                    self.adbase.cancel_timer(self.not_home_timers[device_state])
                    self.not_home_timers[device_state] = None

                self.update_hass_sensor(user_device_sensor, "on")

                if self.hass.get_state(self.somebody_is_home, copy=False) == "off":
                    self.update_hass_sensor(self.somebody_is_home, "on") #somebody is home

                if user_device_sensor in self.all_users_sensors: #check if everyone home
                    if self.hass.get_state(self.everyone_not_home, copy=False) == "on":
                        self.update_hass_sensor(self.everyone_not_home, "off")

                    self.adbase.run_in(self.check_home_state, 2, check_state = "is_home")

            else:
                self.adbase.log("Device State: {}, User Device Sensor: {}, New: {}, State: {}".format(device_state, user_device_sensor, new, self.hass.get_state(user_device_sensor, copy=False)), level = "DEBUG")
                if self.not_home_timers[device_state] == None and self.hass.get_state(user_device_sensor, copy=False) != "off" and int(new) == 0: #run the timer
                    self.run_arrive_scan() #run so it does another scan before declaring the user away as extra check within the timeout time
                    self.not_home_timers[device_state] = self.adbase.run_in(self.not_home_func, self.timeout, device_state = device_state)
                    self.adbase.log("Timer Started for {}".format(device_state), level = "DEBUG")

    def not_home_func(self, kwargs):
        device_state = kwargs["device_state"]
        user_device_sensor = "binary_sensor." + device_state
        user_conf_sensors = self.home_state_entities[device_state]
        sensor_res = list(map(lambda x: self.hass.get_state(x, copy=False), user_conf_sensors))
        sensor_res = [i for i in sensor_res if i != "unknown"] # remove unknown vales from list
        self.adbase.log("Device State: {}, Sensors: {}".format(device_state, sensor_res), level = "DEBUG")

        if  all(list(map(lambda x: int(x) < self.minimum_conf, sensor_res))): #still confirm for the last time
            self.update_hass_sensor(user_device_sensor, "off")

            if user_device_sensor in self.all_users_sensors: #check if everyone not home
                """since at least someone not home, set to off the everyone home state"""
                self.update_hass_sensor(self.everyone_home, "off")

                self.adbase.run_in(self.check_home_state, 2, check_state = "not_home")

        self.not_home_timers[device_state] = None

    def send_mqtt_message(self, kwargs):
        topic = kwargs["topic"]
        payload = kwargs["payload"]
        if kwargs["scan_type"] == "Depart":
            count = kwargs["count"]
            self.gateway_timer = None #meaning no more gateway based timer is running

            if self.mqtt.get_state(self.monitor_entity) == "idle": #meaning its not busy
                self.mqtt.mqtt_publish(topic, payload) #send to scan for departure of anyone
                if count <= self.args.get("depart_scans", 3): #scan for departure times. 3 as default
                    count = count + 1
                    self.run_depart_scan(count = count)

            else: #meaning it is busy so re-run timer for it to get idle before sending the message to start scan
                self.run_depart_scan(delay = 10, count = count)

        elif kwargs["scan_type"] == "Arrive":
            self.mqtt.mqtt_publish(topic, payload) #send to scan for arrival of anyone

        elif kwargs["scan_type"] == "System":
            self.mqtt.mqtt_publish(topic, payload) #just send the data

    def update_hass_sensor(self, sensor, data):
        self.adbase.log("__function__: Entity_ID: {}, Data: {}".format(sensor, data), level = "DEBUG")
        sensorState = self.hass.get_state(sensor, attribute = "all")
        state = sensorState["state"]
        attributes = sensorState["attributes"]

        if state != data:
            state = data
            self.hass.set_state(sensor, state = state, attributes = attributes)

    def gateway_opened(self, entity, attribute, old, new, kwargs):
        """one of the gateways was opened or closed and so needs to check what happened"""
        self.adbase.log("Gateway Sensor {} now {}".format(entity, new), level="DEBUG")

        if self.gateway_timer != None: #meaning a timer is running already
            self.adbase.cancel_timer(self.gateway_timer)
            self.gateway_timer = None

        if self.hass.get_state(self.everyone_not_home, copy=False) == "on": #meaning no one at home
            self.run_arrive_scan()

        elif self.hass.get_state(self.everyone_home, copy=False) == "on": #meaning everyone at home
            self.run_depart_scan()
            #self.run_depart_scan(delay = 90)

        else:
            self.run_arrive_scan()
            self.run_depart_scan()
            #self.run_depart_scan(delay = 90)

    def motion_detected(self, entity, attribute, old, new, kwargs):
        """motion detected somewhere in the house, so needs to check for where users are"""
        self.adbase.log("Motion Sensor {} now {}".format(entity, new), level="DEBUG")

        if self.motion_timer != None: #meaning a timer is running already
            self.adbase.cancel_timer(self.motion_timer)
            self.motion_timer = None

        """ "duraction" parameter could be used in listen_state.
            But need to use a single timer for all motion sensors,
            to avoid running the scan too many times"""
        self.motion_timer = self.adbase.run_in(self.run_rssi_scan, self.args.get("rssi_timeout", 60))

    def check_home_state(self, kwargs):
        check_state = kwargs["check_state"]
        if check_state == "is_home":
            """ now run to check if everyone is home since a user is home"""
            user_res = list(map(lambda x: self.hass.get_state(x, copy=False), self.all_users_sensors))
            user_res = [i for i in user_res if i != "unknown"] # remove unknown vales from list
            user_res = [i for i in user_res if i != None] # remove None vales from list

            if all(list(map(lambda x: x == "on", user_res))): #meaning every one is home
                self.update_hass_sensor(self.everyone_home, "on")

            elif any(list(map(lambda x: x == "on", user_res))): #meaning at least someone is home
                if self.hass.get_state(self.somebody_is_home, copy=False) == "off":
                    self.update_hass_sensor(self.somebody_is_home, "on") #somebody is home

        elif check_state == "not_home":
            """ now run to check if everyone is not home since a user is not home"""
            user_res = list(map(lambda x: self.hass.get_state(x, copy=False), self.all_users_sensors))
            user_res = [i for i in user_res if i != "unknown"] # remove unknown vales from list
            user_res = [i for i in user_res if i != None] # remove None vales from list

            if all(list(map(lambda x: x == "off", user_res))): #meaning no one is home
                self.update_hass_sensor(self.everyone_not_home, "on")

                if self.hass.get_state(self.somebody_is_home, copy=False) == "on":
                    self.update_hass_sensor(self.somebody_is_home, "off") #somebody is home
            else:
                if self.hass.get_state(self.somebody_is_home, copy=False) == "off":
                    self.update_hass_sensor(self.somebody_is_home, "on") #somebody is home

    def reload_device_state(self, kwargs):
        topic = "{}/KNOWN DEVICE STATES".format(self.presence_topic) #get latest states
        self.adbase.run_in(self.send_mqtt_message, 0, topic=topic, payload="", scan_type="System")

    def monitor_changed_state(self, entity, attribute, old, new, kwargs):
        scan = kwargs["scan"]
        topic = kwargs["topic"]
        payload = kwargs["payload"]
        self.adbase.run_in(self.send_mqtt_message, 1, topic = topic, payload = payload, scan_type = "Arrive") #send to scan for arrival of anyone
        self.adbase.cancel_listen_state(self.monitor_handlers[scan])
        self.monitor_handlers[scan] = None

    def run_arrive_scan(self, **kwargs):
        topic = "{}/scan/arrive".format(self.presence_topic)
        payload = ""

        """used to listen for when the monitor is free, and then send the message"""
        if self.mqtt.get_state(self.monitor_entity, copy=False) == "idle": #meaning its not busy
            self.mqtt.mqtt_publish(topic, payload) #send to scan for arrival of anyone
        else:
            """meaning it is busy so wait for it to get idle before sending the message"""
            scan_type = self.mqtt.get_state(self.monitor_entity, attribute="scan_type", copy=False)
            if self.monitor_handlers.get("Arrive Scan", None) == None and scan_type != "arrival": #meaning its not listening already, and arrival not running now
                self.monitor_handlers["Arrive Scan"] = self.mqtt.listen_state(self.monitor_changed_state, self.monitor_entity,
                            new = "idle", old = "scanning", scan = "Arrive Scan", topic = topic, payload = payload)

    def run_depart_scan(self, **kwargs):
        delay = kwargs.get("delay", self.depart_check_time)
        count = kwargs.get("count", 1)

        topic ="{}/scan/depart".format(self.presence_topic)
        payload = ""

        if self.gateway_timer != None: #meaning a timer running aleady
            self.adbase.cancel_timer(self.gateway_timer) #just extra check, shouldn't be needed

        self.gateway_timer = self.adbase.run_in(self.send_mqtt_message, delay, topic = topic,
                        payload = payload, scan_type = "Depart", count = count) #send to scan for departure of anyone

    def run_rssi_scan(self, kwargs):
        topic = "{}/scan/rssi".format(self.presence_topic)
        self.adbase.run_in(self.send_mqtt_message, 0, topic=topic, payload="", scan_type="System")
        self.motion_timer = None

    def restart_device(self, kwargs):
        topic = "{}/scan/restart".format(self.presence_topic)
        payload = ""
        self.mqtt.mqtt_publish(topic, payload) #instruct to restart service

    def clear_location_entities(self, kwargs):
        """used to retrieve the different sensors based on system location, and set them to 0
            this will ensure that if a location goes down and the confidence is set to 100, it doesn"t
            stay that way, and therefore lead to false info"""
        location = kwargs["location"]
        self.adbase.log("Processing System Unavailable for "+ location)
        siteId = location.replace(" ", "_").lower()
        for device_state, entity_list in self.home_state_entities.items():
            for sensor in entity_list:
                if siteId in sensor: #meaning that sensor belongs to that location
                    self.update_hass_sensor(sensor, 0)
                    device_local = sensor.replace("sensor.", "")
                    appdaemon_entity = "{}.{}".format(self.presence_topic, device_local)
                    self.mqtt.set_state(appdaemon_entity, state = 0, rssi = "-99")

        if location in self.location_timers:
            self.location_timers.pop(location)

        entity_id = "{}.{}".format(self.presence_topic, siteId)
        self.mqtt.set_state(entity_id, state = "Offline")

        self.run_arrive_scan()

    def system_state_changed(self, entity, attribute, old, new, kwargs):
        self.adbase.run_in(self.reload_device_state, 0)

    def monitor_scan_now(self, entity, attribute, old, new, kwargs):
        scan_type = self.mqtt.get_state(entity, attribute="scan_type", copy=False)
        locations = self.mqtt.get_state(entity, attribute="locations", copy=False)

        if scan_type == "both":
            self.run_arrive_scan(location=locations)
            self.run_depart_scan(location=locations)

        elif scan_type == "arrival":
            self.run_arrive_scan(location=locations)

        elif scan_type == "depart":
            self.run_depart_scan(location=locations)

        self.mqtt.set_state(entity, state = "idle")

    def load_known_devices(self, kwargs):
        topic = "{}/setup/ADD STATIC DEVICE".format(self.presence_topic)
        timer = 0
        for device in self.args["known_devices"]:
            payload = device
            self.adbase.run_in(self.send_mqtt_message, timer, topic=topic, payload=payload, scan_type="System")
            timer += 15

    def remove_known_device(self, kwargs):
        device = kwargs["device"]
        topic = "{}/setup/DELETE STATIC DEVICE".format(self.presence_topic)
        self.adbase.run_in(self.send_mqtt_message, 0, topic=topic, payload=device, scan_type="System")

    def hass_restarted(self, event, data, kwargs):
        self.setup_global_sensors()
        self.adbase.run_in(self.reload_device_state, 10)

    def setup_global_sensors(self):
        if not self.hass.entity_exists(self.everyone_not_home): #check if the sensor exist and if not create it
            self.adbase.log("Creating Binary Sensor for Everyone Not Home State")
            attributes = {"friendly_name": "Everyone Not Home", "device_class" : "presence"}
            self.hass.set_state(self.everyone_not_home, state = "off", attributes = attributes) #send to homeassistant to create binary sensor sensor for home state

        if not self.hass.entity_exists(self.everyone_home): #check if the sensor exist and if not create it
            self.adbase.log("Creating Binary Sensor for Everyone Home State")
            attributes = {"friendly_name": "Everyone Home", "device_class" : "presence"}
            self.hass.set_state(self.everyone_home, state = "off", attributes = attributes) #send to homeassistant to create binary sensor sensor for home state

        if not self.hass.entity_exists(self.somebody_is_home): #check if the sensor exist and if not create it
            self.adbase.log("Creating Binary Sensor for Somebody is Home")
            attributes = {"friendly_name": "Somebody is Home State", "device_class" : "presence"}
            self.hass.set_state(self.somebody_is_home, state = "off", attributes = attributes) #send to homeassistant to create binary sensor sensor for home state


"""
eklogi Response Utilities

This module contains utility functions for constructing JSON API responses.
"""
import json
import time
import traceback

from flask import Response, request

def response(data, *, success=True, status=200, response_time=None):
    """
    Generates a JSON response given a serializable data object.

    The data object is serialized to JSON to represent the data of the response.

    :param data: the data object to be serialized
    :param success: the flag for whether the response is a success
    :param status: the http response code
    :param response_time the response time
    :return: a JSON response
    """
    # is True may seem redundant, but allows for truthy values to always be
    # serialized to a bool
    req = {'method': request.method,
           'path': request.path,
           'args': request.args,
           'timestamp': int(time.time()),
           'status': status}
    if response_time:
        req['response_time'] = response_time
    res = {'success': success is True,
           'request': req,
           'data': data}
    return Response(response=json.dumps(res, indent=2, separators=(',', ': ')),
                    status=status,
                    mimetype='application/json')

def error_response(error, *, message=None, status=500):
    """
    Generates an error JSON response given an error object.

    The http response code defaults to '500' to represent internal server error.

    Any caught client errors, e.g. malformed requests, should specify the accurate response code.

    :param error: the error object
    :param message: the error message
    :param status: the http response code
    :return: an error JSON response
    """
    res = {'error': type(error).__name__ if isinstance(
        error, BaseException) else error,
           'message': message or ''}
    return response(res, success=False, status=status)

def apply(f):
    """
    Applies a function to return a result that can be returned as a JSON response.

    All errors are caught and returned within a JSON error response.

    :param f: the function to apply
    :return: a success response if no error occurred during applying f, otherwise an error response
    """
    try:
        begin = time.time()
        data = f()
        diff = time.time() - begin
        diff_str = "{}s".format(int(diff)) if int(diff) else \
            "{}ms".format(int(diff * 1000)) if int(diff * 1000) else \
                "{}ns".format(int(diff * 1000000))
        return response(data, response_time=diff_str)
    except Exception as e:  # pylint: disable=broad-except
        traceback.print_exc()
        return error_response(e)


import serial
import _winreg as winreg
import itertools
import time

def enumerate_serial_ports():
    """ Uses the Win32 registry to return a iterator of serial
        (COM) ports existing on this computer.
    """

    path = 'HARDWARE\\DEVICEMAP\\SERIALCOMM'
    try:
        key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, path)
    except WindowsError:
        raise IterationError

    for i in itertools.count():
        try:
            val = winreg.EnumValue(key, i)
            yield (str(val[1]), str(val[0]))
        except EnvironmentError:
            break

def main():

    comPorts=[]
    for i in enumerate_serial_ports():
        comPorts.append(i)

    for i in comPorts:
        print(i)

    com= comPorts[0]

    com= com[0]
    ser = serial.Serial(
                port= com,
                baudrate=115200,
                bytesize=serial.EIGHTBITS,
                timeout=3  )
    while(True):

        ser.flush()
        line="-";
        while 'A' not in line:
            line= ser.readline()
            line.strip();
            print('Looking for A')
            print(line)

        ser.write('list\0')
        time.sleep(1)
        line= ser.readline();
        while(line != "end\n"):
            print(line)
            line= ser.readline()

if __name__ == '__main__':
    main()


from bot import bot
import datetime
import pytz
from utils.database import execute_database_command
from checks.utils import CheckStatus
from users.models import User

@bot.message_handler(content_types=['sticker'])
def get_sticker_id(message):
    print(message.sticker.file_id)

def remove_markup(chat_id, message_id):
    bot.edit_message_reply_markup(chat_id, message_id=message_id, reply_markup=None)

def get_native_datetime(date, time, timezone):
    return pytz.timezone(timezone).localize(
                datetime.datetime.strptime(f'{date} {time}', "%Y-%m-%d %H:%M"), is_dst=None)

def get_user_naming(user, default):
    user_naming = default
    if user.first_name:
        user_naming = user.first_name
        if user.last_name:
            user_naming = user.first_name + ' ' + user.last_name

    return user_naming

def get_schedule(week_days, time_array, timezone):
    user_date_now = datetime.datetime.now(tz=pytz.timezone(timezone)).date()
    schedule_native = []
    for week_day in week_days:
        if week_day > user_date_now.weekday():
            first_day = user_date_now + datetime.timedelta(week_day - user_date_now.weekday())
        else:
            first_day = user_date_now + datetime.timedelta(7 - user_date_now.weekday() + week_day)

        second_day = first_day + datetime.timedelta(7)
        third_day = second_day + datetime.timedelta(7)

        for time in time_array:
            first_datetime_native, second_datetime_native, third_datetime_native = [
                get_native_datetime(str(day), time, timezone) for day in
                [first_day, second_day, third_day]]

            schedule_native.extend([first_datetime_native, second_datetime_native, third_datetime_native])

    schedule_native.sort()
    schedule_utc = [native_datetime.astimezone(pytz.utc) for native_datetime in schedule_native]
    return schedule_native, schedule_utc

def score_users():
    success_checks = execute_database_command('''SELECT h.user_id, h.fine FROM
        checks c JOIN habits h ON c.habit_id = h.id
        WHERE  c.status=%s;
        ''', (CheckStatus.SUCCESS.name, ))[0]
    for success_check in success_checks:
        u = User.get(success_check[0])
        u.score += success_check[1]
        u.save()

    fail_checks = execute_database_command('''SELECT h.user_id, h.fine FROM
            checks c JOIN habits h ON c.habit_id = h.id
            WHERE  c.status=%s;
            ''', (CheckStatus.FAIL.name,))[0]
    for fail_check in fail_checks:
        u = User.get(fail_check[0])
        u.score -= fail_check[1]
        u.save()

def send_info_to_users():
    users = execute_database_command('SELECT id, first_name, language_code FROM users;')[0]

    for user in users:
        ru_text = f'Привет{", " + user[1] if user[1] else ""}!\n\n' \
                  f'Я решил больше не церемониться. Теперь разговариваю дерзко и на "ты". ' \
                  f'Может, хотя бы это поможет тебе побороть свою лень. ' \
                  f'А ещё я теперь отправляю стикеры.\n\n' \
                  f'Короче, я теперь не унылый стандартный бот.'
        en_text = f'Hello{", " + user[1] if user[1] else ""}!\n\n' \
                  f'I decided to no longer stand on ceremony. Now I speak impertinently. ' \
                  f'Maybe at least it will help you overcome your laziness. ' \
                  f'And now I send stickers.\n\n' \
                  f'In short, I am no longer a dull standard bot.'
        text = ru_text if user[2] == 'ru' else en_text

        try:
            bot.send_message(user[0], text, parse_mode='Markdown')
            bot.send_sticker(user[0], 'CAADAgADcgADE-ZSAdekz3zzPdFUAg')
        except Exception as e:
            print(e)


import codecs
from numpy import *
import matplotlib.pyplot as plt

def load_data(path):
	"""
	@brief      Loads a data
	@param      path  The path
	@return     data set
	"""
	data_set = list()
	with codecs.open(path) as f:
		for line in f.readlines():
			data = line.strip().split(" ")
			flt_data = list(map(double, data))
			data_set.append(flt_data)
	return data_set

def dist_eucl(vecA, vecB):
	"""
	@brief      the similarity function
	@param      vecA  The vector a
	@param      vecB  The vector b
	@return     the euclidean distance
	"""
	return sqrt(sum(power(vecA - vecB, 2)))

def rand_cent(data_mat, k):
	"""
	@brief      select random centroid
	@param      data_mat  The data matrix
	@param      k
	@return     centroids
	"""
	n = shape(data_mat)[1]
	centroids = mat(zeros((k, n)))
	for j in range(n):
		minJ = min(data_mat[:,j])
		rangeJ = float(max(data_mat[:,j]) - minJ)
		centroids[:,j] = mat(minJ + rangeJ * random.rand(k,1))
	return centroids

def kMeans(data_mat, k, dist = "dist_eucl", create_cent = "rand_cent"):
	"""
	@brief      kMeans algorithm
	@param      data_mat     The data matrix
	@param      k            num of cluster
	@param      dist         The distance funtion
	@param      create_cent  The create centroid function
	@return     the cluster
	"""
	m = shape(data_mat)[0]
	# 初始化点的簇
	cluster_assment = mat(zeros((m, 2)))  # 类别，距离
	# 随机初始化聚类初始点
	centroid = eval(create_cent)(data_mat, k)
	cluster_changed = True
	# 遍历每个点
	while cluster_changed:
		cluster_changed = False
		for i in range(m):
			min_index = -1
			min_dist = inf
			for j in range(k):
				distance = eval(dist)(data_mat[i, :], centroid[j, :])
				if distance < min_dist:
					min_dist = distance
					min_index = j
			if cluster_assment[i, 0] != min_index:
				cluster_changed = True
				cluster_assment[i, :] = min_index, min_dist**2
		# 计算簇中所有点的均值并重新将均值作为质心
		for j in range(k):
			per_data_set = data_mat[nonzero(cluster_assment[:,0].A == j)[0]]
			centroid[j, :] = mean(per_data_set, axis = 0)
	return centroid, cluster_assment

def kmeans_plot_cluster(data_mat, cluster_assment, centroid):
	"""
	@brief      plot cluster and centroid
	@param      data_mat        The data matrix
	@param      cluster_assment  The cluste assment
	@param      centroid        The centroid
	@return
	"""
	plt.figure(figsize=(15, 6), dpi=80)
	plt.subplot(121)
	plt.plot(data_mat[:, 0], data_mat[:, 1], 'o')
	plt.title("source data", fontsize=15)
	plt.subplot(122)
	k = shape(centroid)[0]
	colors = [plt.cm.get_cmap("Spectral")(each) for each in linspace(0, 1, k)]
	for i, col in zip(range(k), colors):
	    per_data_set = data_mat[nonzero(cluster_assment[:,0].A == i)[0]]
	    plt.plot(per_data_set[:, 0], per_data_set[:, 1], 'o', markerfacecolor=tuple(col),
	             markeredgecolor='k', markersize=10)
	for i in range(k):
		plt.plot(centroid[:,0], centroid[:,1], '+', color = 'k', markersize=18)
	plt.title("K-Means Clustering, k = "+str(k), fontsize=15)
	plt.show()

if __name__ == '__main__':
	data_mat = mat(load_data("D:\\大三下\\人工智能基础\\Thecode\\Chapter2\\data\\five_cluster.txt"))
	data_mat = data_mat[:,1:3]

	centroid, cluster_assment = kMeans(data_mat, 5)
	sse = sum(cluster_assment[:,1])
	print("SSE is ", sse)
	kmeans_plot_cluster(data_mat, cluster_assment, centroid)


"""
Import deliveryfleet.csv file

Here we need Two driver features: mean distance driven per day
(Distance_feature) and the mean percentage of time a driver was >5 mph over
the speed limit (speeding_feature).

Perform K-means clustering to distinguish urban drivers and rural drivers.

Perform K-means clustering again to further distinguish speeding drivers from
those who follow speed limits, in addition to the rural vs. urban division.

Label accordingly for the 4 groups.

file_name - "deliveryfleet.py"

"""

# K-Means Clustering

# Importing the libraries
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('deliveryfleet.csv')
X = dataset.iloc[:, [1, 2]].values

# Using the elbow method to find the optimal number of clusters
from sklearn.cluster import KMeans
wcss = []
for i in range(1, 11):
    kmeans = KMeans(n_clusters = i, init = 'k-means++', random_state = 0)
    kmeans.fit(X)
    wcss.append(kmeans.inertia_)
plt.plot(range(1, 11), wcss)
plt.title('The Elbow Method')
plt.xlabel('Number of clusters')
plt.ylabel('WCSS')
plt.show()

# Fitting K-Means to the dataset
kmeans = KMeans(n_clusters = 2, init = 'k-means++', random_state = 0)
y_kmeans = kmeans.fit_predict(X)

# Visualising the clusters
plt.scatter(X[y_kmeans == 0, 0], X[y_kmeans == 0, 1], s = 100, c = 'red', label = 'Rural Drivers')
plt.scatter(X[y_kmeans == 1, 0], X[y_kmeans == 1, 1], s = 100, c = 'blue', label = 'Urban Drivers')
plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s = 300, c = 'yellow', label = 'Centroids')
plt.title('Clusters of Drivers')
plt.xlabel('Distance Feature')
plt.ylabel('Speeding Feature')
plt.legend()
plt.show()

# Fitting K-Means to the dataset (for Rash and Safe Drivers)
kmeans = KMeans(n_clusters = 4, init = 'k-means++', random_state = 0)
y_kmeans = kmeans.fit_predict(X)

# Visualising the clusters
plt.scatter(X[y_kmeans == 0, 0], X[y_kmeans == 0, 1], s = 100, c = 'red', label = 'Rural Safe Drivers')
plt.scatter(X[y_kmeans == 1, 0], X[y_kmeans == 1, 1], s = 100, c = 'blue', label = 'Urban Safe Drivers')
plt.scatter(X[y_kmeans == 2, 0], X[y_kmeans == 2, 1], s = 100, c = 'green', label = 'Urban Rash Drivers')
plt.scatter(X[y_kmeans == 3, 0], X[y_kmeans == 3, 1], s = 100, c = 'cyan', label = 'Rural Rash Drivers')
plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s = 300, c = 'yellow', label = 'Centroids')
plt.title('Clusters of Drivers')
plt.xlabel('Distance Feature')
plt.ylabel('Speeding Feature')
plt.legend()
plt.show()


import torchvision
from torch.utils.data import random_split, DataLoader
from pytorch_lightning import LightningDataModule
import typing as tp

class CelebaDataModule(LightningDataModule):

    def __init__(
            self,
            data_dir: str = "celeba",
            image_shape: int = (128, 128),
            batch_size: int = 32,
            num_workers: int = 32,
            validation_len: tp.Union[int, float] = 500,
            attributes: tp.List[str] = ['Black_Hair', 'Blond_Hair', 'Brown_Hair', 'Male', 'Young'],
            **kwargs
    ):
        self.transforms = torchvision.transforms.Compose([
            torchvision.transforms.Resize(image_shape),
            torchvision.transforms.ToTensor(),
            torchvision.transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
        ])
        attrs = '5_o_Clock_Shadow ' \
                'Arched_Eyebrows ' \
                'Attractive ' \
                'Bags_Under_Eyes ' \
                'Bald ' \
                'Bangs ' \
                'Big_Lips ' \
                'Big_Nose ' \
                'Black_Hair ' \
                'Blond_Hair ' \
                'Blurry ' \
                'Brown_Hair ' \
                'Bushy_Eyebrows ' \
                'Chubby ' \
                'Double_Chin ' \
                'Eyeglasses ' \
                'Goatee ' \
                'Gray_Hair ' \
                'Heavy_Makeup ' \
                'High_Cheekbones ' \
                'Male ' \
                'Mouth_Slightly_Open ' \
                'Mustache ' \
                'Narrow_Eyes ' \
                'No_Beard ' \
                'Oval_Face ' \
                'Pale_Skin ' \
                'Pointy_Nose ' \
                'Receding_Hairline ' \
                'Rosy_Cheeks ' \
                'Sideburns ' \
                'Smiling ' \
                'Straight_Hair ' \
                'Wavy_Hair ' \
                'Wearing_Earrings ' \
                'Wearing_Hat ' \
                'Wearing_Lipstick ' \
                'Wearing_Necklace ' \
                'Wearing_Necktie ' \
                'Young'
        attr2index = {j: i for i, j in enumerate(attrs.split())}
        self.attributes = attributes
        self.label_subset_indices = [attr2index[j] for j in self.attributes]
        super().__init__()
        self.data_dir = data_dir
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.validation_len = validation_len
        self.image_shape = image_shape

    def target_transform(self, label):
        return label[self.label_subset_indices]

    def setup(self, stage=None):
        self.celeba = torchvision.datasets.CelebA(self.data_dir, target_type='attr', transform=self.transforms,
                                                  target_transform=self.target_transform, download=True)
        if isinstance(self.validation_len, float):
            self.validation_len = int(self.validation_len * len(self.celeba))
        self.train_dataset, self.val_dataset = random_split(self.celeba,
                                                            lengths=[len(self.celeba) - self.validation_len,
                                                                self.validation_len])

    def train_dataloader(self):
        return DataLoader(self.train_dataset, self.batch_size, shuffle=True,
                          num_workers=self.num_workers)

    def val_dataloader(self):
        return DataLoader(self.val_dataset, self.batch_size, shuffle=False,
                          num_workers=self.num_workers)

    # def test_dataloader(self) -> Union[DataLoader, List[DataLoader]]:
    #     pass


import numpy as np
import pandas as pd
from pandas import Series, DataFrame
import matplotlib.pyplot as plt
from arch.unitroot import ADF
import statsmodels.api as sm
import re
import datetime
from jqdata import jy
from numpy import nan

# 数据引擎
class DataEngine():
    # 获取tick数据
    def getTickData(self, symbol, start_dt, end_dt):
        data = get_ticks(symbol, start_dt=start_dt, end_dt=end_dt,
                         fields=['time', 'current', 'volume', 'money', 'position', 'a1_v', 'a1_p', 'b1_v', 'b1_p'])
        df1 = DataFrame(data)
        # 时间标准化，方便后续时间运算
        df1.time = pd.to_datetime(df1.time, format='%Y%m%d%H%M%S.%f')
        df1.index = df1.time
        # 去掉重复索引
        df1.drop_duplicates(subset='time', keep='last', inplace=True)
        # 去掉多余列
        df1.drop(columns='time', inplace=True)
        return df1

    # 取前一周的Tick数据
    def getPreWeekTickData(self, symbol):
        e = datetime.datetime.now()
        end_dt = e.strftime('%Y-%m-%d')
        s = e - datetime.timedelta(days=7)
        start_dt = s.strftime('%Y-%m-%d')
        return self.getTickData(symbol, start_dt=start_dt, end_dt=end_dt)

    # 获取某个合约的前一周的秒级数据

    # 由tick数据转换为秒级数据
    def getPreWeekSecData(self, symbol, freq=1):
        df1 = self.getPreWeekTickData(symbol)
        df1 = df1.current.resample('%dS' % freq).agg(['last', 'first', 'max', 'min']).dropna()
        # 筛选固定交易时间数据，只做日内，并且不做夜盘
        # 这里的数据筛选是特定的针对性数据
        d1 = df1.loc[datetime.time(9, 0, 0):datetime.time(10, 15, 0), :]
        d2 = df1.loc[datetime.time(10, 30, 0):datetime.time(11, 30, 0), :]
        d3 = df1.loc[datetime.time(13, 30, 0):datetime.time(15, 0, 0), :]
        # 连接函数
        return pd.concat([d1, d2, d3])

    def getData(self):
        # 可以得到不同时间 不同频率的所有主力合约数据
        pass

    # 获取所有主力合约信息
    def getAllMainContract(self):
        secs = get_all_securities(types=['futures'])
        main_secs = secs[secs.display_name.apply(lambda x: '主力合约' in x)]
        main_secs_code = [get_dominant_future(s[0:-4]) for s in main_secs.name if get_dominant_future(s[0:-4])]
        return secs.loc[main_secs_code, :]

    # 获取所有主力合约上周的tick数据
    # 每日执行一次，方便分析数据时，快速加载数据
    def getPreWeekAllTickData(self):
        e = datetime.datetime.now()
        end_dt = e.strftime('%Y-%m-%d')
        s = e - datetime.timedelta(days=7)
        start_dt = s.strftime('%Y-%m-%d')
        codes = self.getAllMainContract()
        d = {}
        for i in codes.index:
            d[i] = self.getTickData(i, start_dt=start_dt, end_dt=end_dt)
        return pd.Panel(d)

    # 获取某个合约的合约乘数  计数单位 和 最小变动单位
    # 犯了一个基本错误，忘了写self，报了一堆看不懂的错误，郁闷
    def future_basic_info(self, future):
        # 这个后续改成从本地的meta数据中拿metadata数据的方法，暂时先这么使用
        if "9999" in future or "8888" in future:
            match = re.match(r"(?P<underlying_symbol>[A-Z]{1,})", future)
            if not match:
                raise ValueError("未知期货标的：{}".format(future))
            else:
                future = get_dominant_future(match.groupdict()["underlying_symbol"])
        code = future.split(".")[0]
        q = query(jy.Fut_ContractMain).filter(jy.Fut_ContractMain.ContractCode == code)
        result = jy.run_query(query_object=q).to_dict("record")
        if result:
            result = result.pop()
            min_point = re.match("(?P<value>^[0-9]+([.]{1}[0-9]+){0,1})", result["LittlestChangeUnit"]).groupdict(nan)[
                "value"]
            return {"ContractUnit": result["CMValue"],
                    "PriceScale": float(str(min_point)[:-1] + "1") if float(min_point) < 1 else 1,
                    "MinPoint": float(min_point)}
        else:
            cu = nan
            current_code = code[:-4].upper()
            if trade_units.get(current_code):
                cu = trade_units.get(current_code)
            return {"ContractUnit": cu,
                    "PriceScale": nan,
                    "MinPoint": nan}

    # 查询交易单位，输入参数为合约的代码 比如RU1909.XSGE合约，这里输入RU
    def get_trade_units(self, symbol):
        code = symbol.split(".")[0][:-4].upper()
        return trade_units.get(code)

    # 绘制时间序列数据图，过滤掉中间空白情况
    def plot(self, dataSeries):
        pass

# 计算引擎
class StatisEngine():
    def integration(self, priceX):
        # 一阶单整计算
        logX = priceX
        retX = logX.diff()[1:]
        adfLX = ADF(logX)
        adfRX = ADF(retX)
        return adfLX.pvalue, adfRX.pvalue

    # 判断数据是否为一阶单整序列
    def isIntegration(self, priceX):
        # 对数价格序列是非平稳时间序列
        # 对数差分价格是平稳时间序列
        pLX, pRX = self.integration(priceX)
        # 对数价格置信度大于0.05才认为是非平稳时间序列
        # 差分序列置信度小于0.01才认为是平稳时间序列
        if ((pLX > 0.05) and (pRX < 0.05)):
            # 是一阶单整序列
            return True
        else:
            # 不是一阶单整序列
            return False

    # 判断两个序列是否有协整效应
    def isCointegration(self, priceX, priceY):
        logX = priceX
        logY = priceY
        results = sm.OLS(logY, sm.add_constant(logX)).fit()
        resid = results.resid
        adfSpread = ADF(resid)
        if adfSpread.pvalue >= 0.05:
            # 残差序列是非平稳时间序列，不具有协整关系
            return (False, results.params[0], results.params[1])
        else:
            # 残差序列是平稳时间序列，具有协整关系
            return (True, results.params[0], results.params[1])

    # 获取残差序列
    def getSpread(self, priceX, priceY, alpha, beta):
        return priceY - alpha - beta * priceX

# 一阶单整检验筛选
def filter_integration_datas(ac):
    de = DataEngine()
    se = StatisEngine()
    print("1. 获取主力合约基本面数据")
    validData = {}
    print("2. 开始一阶单整检验")
    for i in ac.index:
        # 获取经过处理的数据
        print("%s 合约-------------1. 错误合约检验" % str(i))
        if i not in error_stocks:
            print("%s 合约-------------2. 获取秒级数据" % str(i))
            data1 = de.getPreWeekSecData(i)
            # 排除没有数据的合约
            print("%s 合约-------------3. 数据量检验,数据量要在10000以上" % str(i))
            if (data1.shape[0] >= 10000):
                # 对秒级 频率数据进行一阶单整检验
                print("%s 合约-------------4. 合约一阶单整检验" % str(i))
                result = se.isIntegration(data1['last'])
                if result == True:
                    print("%s 合约-------------5. 合约一阶单整检验成功" % str(i))
                    # 用收盘价数据乘以最小交易单位作为要分析的数据
                    last_data = data1['last']
                    cu = de.get_trade_units(str(i))
                    print("%s 合约-------------6. cu:%s" % (str(i), str(cu)))
                    validData[i] = (last_data * cu)
    data = DataFrame(validData)
    return data.dropna(how='all')

# 协整效应筛选
def filter_cointegration(ac, data):
    l = data.columns.size
    se = StatisEngine()
    pairs = []
    for i in range(l):
        for j in range(i + 1, l):
            s1 = data.columns[i]
            s2 = data.columns[j]
            # 前项和后项填充都用上，防止出现空值
            t1 = data.loc[:, [s1, s2]].dropna(how='all').bfill().ffill()
            d1 = t1[s1]
            d2 = t1[s2]
            print("套利对：(%s,%s)开始协整检验" % (s1, s2))
            (result, alpha, beta) = se.isCointegration(d1, d2)
            if result:
                print("套利对为：(%s,%s),alpha:%s,beta:%s"
                      % (ac.loc[s1, :].display_name, ac.loc[s2, :].display_name, str(alpha), str(beta)))
                pairs.append((s1, s2, alpha, beta))
    return pairs

e = datetime.datetime.now()
end_date = e.strftime('%Y-%m-%d')
s = e - datetime.timedelta(days=7)
start_date = s.strftime('%Y-%m-%d')

def Main(sec='1s', start_date=start_date, end_date=end_date):
    # todo 改造为 可以传送频率 时间段 和 具体
    # 获取主力合约基本面数据
    de = DataEngine()
    ac = de.getAllMainContract()
    # 一阶单整筛选
    data = filter_integration_datas(ac)
    # 协整检验
    pairs = filter_cointegration(ac, data)
    # 对所有套利对生成策略图
    # todo 添加导出报表的功能，一天分析一次，然后做各种比较
    create_report(ac, pairs, data)
    print("协整检验完成")
    return ac, pairs, data

# 目前有问题的合约和无法交易的合约
error_stocks = ['JR2005.XZCE', 'IC1907.CCFX', 'IF1907.CCFX', 'IH1907.CCFX', 'T1909.CCFX', 'TF1909.CCFX', 'TS1909.CCFX']
# 合约代码和交易单位数据，暂时存放在这里
trade_units = {'AP': 10, 'CF': 5, 'CJ': 5, 'CY': 5, 'FG': 20, 'JR': 20, 'LR': 20, 'MA': 10, 'OI': 10, 'PM': 50,
               'RI': 20, 'RM': 10,
               'RS': 10, 'SF': 5, 'SM': 5, 'SR': 10, 'TA': 5, 'WH': 20, 'ZC': 100, 'A': 10, 'AG': 15, 'AL': 5,
               'AU': 1000,
               'B': 10, 'BB': 500, 'BU': 10, 'C': 10, 'CS': 10, 'CU': 5, 'EG': 10, 'FB': 500, 'FU': 10, 'HC': 10,
               'I': 100, 'IC': 200, 'IF': 300, 'IH': 300, 'J': 100, 'JD': 10, 'JM': 60, 'L': 5, 'M': 10, 'NI': 1,
               'P': 10, 'PB': 5, 'PP': 5, 'RB': 10, 'RU': 10, 'SC': 1000, 'SN': 1, 'SP': 10, 'T': 10000, 'TF': 10000,
               'TS': 20000, 'V': 5, 'WR': 10, 'Y': 10, 'ZN': 5}

if __name__ == '__main__':
    # Main()
    print("hello")


#copyright Vaibhav Raj 2018(email_ID:-vaibhav.raj.phe15@itbhu.ac.in,vaibhavraj46@gmail.com)
#
# Licensed under the Apache License, Version 2.0 (the "License");
#==========================================================================================================================

'''
A simple handwritten Devnagari numerals(0-9) classifier for training model.For training we would use the extracted version of the zip file :-'DevanagariHandwrittenCharacterDataset' that I've already uploaded.The extracted version needs to be saved along with the 'image_recog_train2.py'(file containing code for training the model) and 'image_recog_test2.py'(file containing code for classifying any random test image).
'''

#import modules
from PIL import Image
import numpy as np
import tensorflow as tf
import os
import re

'''
Algorithm to create two lists:-
(1)'image_batch' list for holding image color intensity matrix of all the handwritten devnagari numerals present in the train set of the extracted version of 'DevanagariHandwrittenCharacterDataset'.
(2)'labels' list for holding respective labels of the elements of 'image_batch'.
'''
L=[]
image_data=[]
labels=[]
path='DevanagariHandwrittenCharacterDataset\\DevanagariHandwrittenCharacterDataset\\Train'
l=os.listdir(path)
D=[]

for i in l:
    if(re.search('digit',i)):
        D.append(i)
        m=os.listdir(path+'\\'+i)
        L.append(m)

for j in range(len(m)):
    for k in D:
        d=int(k[::-1][0])
        n=L[d][j]
        imag=Image.open(path+'\\'+k+'\\'+n)
        image=imag.convert(mode='L')
        image_data.append(np.array(image,dtype=np.float32))
        labels.append(d)

image_data = np.multiply(image_data, 1.0/255.0)

image_batch = image_data.reshape(-1, 32,32,1)
labels=np.asarray(labels, dtype=np.int32)

#Creating placeholders:-image_batch_pl(for 'image_batch' list) and labels_pl(for 'labels' list):-
image_batch_pl=tf.placeholder(dtype=tf.float32,shape=[None,32,32,1],name='image_batch_pl')
labels_pl=tf.placeholder(dtype=np.int32,shape=[None])

'''
Creating CNN(convoluted neural network) model:-
'''

#Creating 1st convolution layer:-
conv1 = tf.layers.conv2d(inputs=image_batch_pl,filters=32,kernel_size=[5, 5],padding="same",activation=tf.nn.relu,name='conv1')

#creating 1st pooling layer:-
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2,name='pool1')

#Creating 2nd convolution layer:-
conv2 = tf.layers.conv2d(inputs=pool1,filters=64,kernel_size=[5, 5],padding="same",activation=tf.nn.relu,name='conv2')

#Creating 2nd pooling layer:-
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2,name='pool2')

#Flattening the pool2 matrix into vector:-
pool2_flat = tf.reshape(pool2, [-1, 8 * 8 * 64],name='pool2_flat')

#Creating fully connected layer 'dense' with RELU activation:-
dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu,name='dense')

#Creating dropout layer:-
dropout = tf.layers.dropout(inputs=dense, rate=0.25, training=True,name='dropout')

#Again creatinf a fully connected layer:-
logits = tf.layers.dense(inputs=dropout, units=10,name='logits')

#Applying softmax activation function to 'logits':-
cost=tf.nn.softmax(logits, name='cost')

'''
Model created:-
'''
#Converting the labels into one-hot vector:-
labelV=tf.one_hot(labels_pl,depth=10,name='labelV')

'''
Defining the loss function,optimizer,saver(for saving and restoring the session),batch size,iterations(as iters here),epochs(here its 30) along with initializing a new session and variables:-
'''
loss = -tf.reduce_sum(labelV*tf.log(cost),name='loss')
sess=tf.Session()
init=tf.global_variables_initializer()
sess.run(init)
train=tf.train.GradientDescentOptimizer(0.001).minimize(loss)
saver=tf.train.Saver()
batch_size=50
iter=(len(labels)//50)

'''
Training the model:-
'''
for h in range(20):
	for k in range(iter):
		sess.run(train,feed_dict={image_batch_pl:image_batch[batch_size*k:batch_size*(k+1)],labels_pl:labels[batch_size*k:batch_size*(k+1)]})

	print(("{0} epochs").format(h))

'''
Saving the session as a checkpoint file:-'./image_train2.ckpt' and printing the confirmation message:-
'''
save_path = saver.save(sess, './image_train2.ckpt')
print ("Model saved in file: ", save_path)