#!/usr/bin/env python3
# vim: set expandtab cindent sw=4 ts=4:
#
# (C)2015 Jan Tulak <jan@tulak.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import unittest
from unittest import mock as mock
import json
import dummy
import huefri
import huefri.common
from huefri.tradfri import Tradfri

class TestTradfri(unittest.TestCase):

    def setUp(self):
        self.fnt_log = huefri.common.log
        huefri.common.log = lambda x,y: None
        huefri.tradfri.log = lambda x,y: None
        huefri.common.Config._config = json.loads("""{
            "hue":{
                "addr":"hue",
                "secret": "SECRET1",
                "controlled": [1,2,3],
                "main": 1
                },
            "tradfri":{
                "addr": "tradfri",
                "secret": "SECRET2",
                "controlled": [0,1,2],
                "main": 0
                }
            }""")
        self.map = [
            {"hex": "efd275", # warm - 34,98 in OpenHab
                "hsb": {'on': True, 'hue':  6188, 'sat': 249}},
            {"hex": "f1e0b5", # medium - 42,59 in OpenHab
                "hsb": {'on': True, 'hue':  7644, 'sat': 150}},
            {"hex": "f5faf6", # cold - 216,5 in OpenHab
                "hsb": {'on': True, 'hue': 39312, 'sat':  13}},
        ]
        self.cls_map = huefri.common.COLORS_MAP
        huefri.common.COLORS_MAP = self.map
        with mock.patch('pytradfri.coap_cli.api_factory', dummy.TAPI) as m:
            with mock.patch('pytradfri.gateway.Gateway', dummy.Gateway) as n:
                self.tradfri = Tradfri.autoinit()

    def tearDown(self):
        huefri.common.log = self.fnt_log
        huefri.common.COLORS_MAP = self.cls_map

    def test_init(self):
        with mock.patch('pytradfri.coap_cli.api_factory', dummy.TAPI) as m:
            with mock.patch('pytradfri.gateway.Gateway', dummy.Gateway) as n:
                tradfri = Tradfri.autoinit()
        self.assertEqual('tradfri', tradfri.api.ip)
        self.assertIsNone(tradfri.color)
        self.assertTrue(isinstance(tradfri.gateway, dummy.Gateway))

    def test__set(self):
        self.tradfri._set(0, "caffee", 100)
        self.assertEqual("caffee", self.tradfri.gateway.lights[0].color)
        self.assertEqual(100, self.tradfri.gateway.lights[0].dimmer)
        self.assertTrue(self.tradfri.gateway.lights[0].state)

        self.tradfri._set(0, "caffee", 0)
        self.assertEqual("caffee", self.tradfri.gateway.lights[0].color)
        self.assertFalse(self.tradfri.gateway.lights[0].state)

        # we didn't changed any other light
        self.assertIsNone(self.tradfri.gateway.lights[1].color)

    def test_set_all(self):
        self.tradfri.set_all("bababa", 150)
        self.assertEqual("bababa", self.tradfri.gateway.lights[0].color)
        self.assertEqual("bababa", self.tradfri.gateway.lights[1].color)
        self.assertEqual("bababa", self.tradfri.gateway.lights[2].color)
        self.assertIsNone(self.tradfri.gateway.lights[3].color)

        self.assertEqual(150, self.tradfri.gateway.lights[0].dimmer)
        self.assertTrue(self.tradfri.gateway.lights[2].state)

    def test_changed(self):
        # exception if we don't know about hue
        self.tradfri.hue = None
        with self.assertRaises(Exception):
            self.tradfri.changed()

        # set up
        self.tradfri.set_all("bababa", 150)
        self.tradfri.hue = dummy.DummyHub()

        # save current state
        self.tradfri.hue.set_time_to_now()
        self.assertFalse(self.tradfri.changed())
        # move time, test if it remembers state
        self.tradfri.hue.set_time_to_past()
        self.assertFalse(self.tradfri.changed())

        # change state
        self.tradfri.set_all("caffee", 100)
        self.assertTrue(self.tradfri.changed())
        # move time, test if it remembers state
        self.tradfri.hue.set_time_to_past()
        self.assertFalse(self.tradfri.changed())

    def test_update(self):
        self.tradfri.hue = dummy.DummyHub()

        # the colors of tradfri should change
        with mock.patch('huefri.tradfri.Tradfri.changed', lambda x: True) as m:
            self.tradfri.set_all("efd275", 100)
            self.tradfri.update()
            self.assertEqual({'on': True, 'hue':  6188, 'sat': 249, 'bri': 100}, self.tradfri.hue.hsb)

        # the colors of tradfri should stay same as in the previous case
        with mock.patch('huefri.tradfri.Tradfri.changed', lambda x: False) as m:
            self.tradfri.set_all("f5faf6", 150)
            self.tradfri.update()
            self.assertEqual({'on': True, 'hue':  6188, 'sat': 249, 'bri': 100}, self.tradfri.hue.hsb)


import jsonlines
import os

# data_path = "D:\IUSTIN\DoinaData\data_splits\humanitarian_orig"
# in_file1 = "train_bt_en-de-en.jsonl"
# in_file2 = "train_bt_en-de-fr-en.jsonl"
# out_file = "train_bt.jsonl"

data_path = r"D:/IUSTIN/DoinaData/data_splits/unlabeled"
in_file1 = "unlabeled_bt_en-de-en.jsonl"
in_file2 = "unlabeled_bt_en-de-fr-en.jsonl"
out_file = "unlabeled_bt.jsonl"

def main():

    with jsonlines.open(os.path.join(data_path, in_file1)) as f1, \
        jsonlines.open(os.path.join(data_path, in_file2)) as f2, \
        jsonlines.open(os.path.join(data_path, out_file), 'w') as f_out:
            for obj1, obj2 in zip(f1, f2):
                assert obj1['id'] == obj2['id']
                obj = {**obj1, **obj2}
                f_out.write(obj)

if __name__ == "__main__":
    main()


import torch
import torch.nn as nn
import torch.optim as optim
from src.utils.images import load_traning_images
from torch import device, cuda
from src.models.gender_classifier import GenderClassifier
from src.core.config import settings

def train_model():
    device('gpu') if cuda.is_available() else device('cpu')
    images, labels = load_traning_images()
    dataset = torch.utils.data.TensorDataset(images, labels)
    data_loader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)
    model = GenderClassifier()
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=0.001)
    num_epochs = settings.NUM_EPOCH
    print(type(num_epochs))
    for epoch in range(num_epochs):
        running_loss = 0.0
        for inputs, labels in data_loader:
            optimizer.zero_grad()
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.item() * inputs.size(0)
        epoch_loss = running_loss / len(dataset)
        print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, epoch_loss))
    return model


import requests
import json

def send_fcm_token_for_device(serverToken, deviceToken, order_id, status):

  headers = {
        'Content-Type': 'application/json',
        'Authorization': 'key=' + serverToken,
      }

  body = {
          'notification': {'title': 'Your order #' + order_id +' status : '+status,
                            'body': 'Thank you for your purchase'
                            },
          'to':
              '/topics/all', ##for all /topics/all , for individual deviceToken is passed
          'priority': 'high',
        #   'data': dataPayLoad,
        }
  response = requests.post("https://fcm.googleapis.com/fcm/send",headers = headers, data=json.dumps(body))
  print(response.status_code)

  print(response.json())


import numpy as np
import pandas as pd
import ccm
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import LassoCV
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import mutual_info_classif
from sklearn.feature_selection import SelectFromModel
def mutual_mutual(data,label,k=200):
    model_mutual= SelectKBest(mutual_info_classif, k=k)
    new_data=model_mutual.fit_transform(data, label)
    mask=model_mutual.get_support(indices=True)
    return new_data,mask
def elasticNet(data,label,alpha =np.array([0.01]),l1_ratio=0.1):
    enet=ElasticNet(alpha=alpha, l1_ratio=l1_ratio)
    enet.fit(data,label)
    mask_ = enet.coef_
    mask=np.nonzero(mask_)
    new_data = data[:,mask]
    return new_data,mask
def lassodimension(data,label,alpha=np.array([0.01,0.02,0.03])):
    lassocv=LassoCV(cv=5, alphas=alpha).fit(data, label)
    x_lasso = lassocv.fit(data,label)
    mask = np.nonzero(x_lasso.coef_)
    new_data = data[:,mask]
    return new_data,mask
def ET(data,label):
    clf=ExtraTreesClassifier(n_estimators=200)
    clf=clf.fit(data,label)
    model=SelectFromModel(clf, prefit=True)
    X_new=model.transform(data)
    return X_new
def CCM_feature(X_res,y_res):
    epsilon = 0.1; num_features = 200; type_Y = 'binary'
    rank = ccm.ccm(X_res, y_res, num_features, type_Y, epsilon, iterations = 50, verbose = True)
    selected_feats = np.argsort(rank)[:200]
    X=X_res[:,selected_feats]
    return X


from langchain.prompts import (
    HumanMessagePromptTemplate,
    SystemMessagePromptTemplate,
    ChatPromptTemplate,
    FewShotChatMessagePromptTemplate,
)

from langchain.chat_models import ChatOpenAI

from llmner.utils import (
    dict_to_enumeration,
    inline_annotation_to_annotated_document,
    align_annotation,
    annotated_document_to_few_shot_example,
    detokenizer,
    annotated_document_to_conll,
)

from llmner.templates import SYSTEM_TEMPLATE_EN

from typing import List, Dict
from llmner.data import (
    AnnotatedDocument,
    AnnotatedDocumentWithException,
    NotContextualizedError,
    Conll,
)

from tqdm import tqdm

from concurrent.futures import ThreadPoolExecutor, as_completed
import multiprocessing

CPU_COUNT = multiprocessing.cpu_count()

import logging

logger = logging.getLogger(__name__)

class BaseNer:
    """Base NER model class. All NER models should inherit from this class."""

    def __init__(
        # TODO: add env variables
        self,
        model: str = "gpt-3.5-turbo",
        max_tokens: int = 256,
        stop: List[str] = ["###"],
        temperature: float = 1.0,
        model_kwargs: Dict = {},
    ):
        """NER model. Make sure you have at least the OPENAI_API_KEY environment variable set with your API key. Refer to the python openai library documentation for more information.

        Args:
            model (str, optional): Model name. Defaults to "gpt-3.5-turbo".
            max_tokens (int, optional): Max number of new tokens. Defaults to 256.
            stop (List[str], optional): List of strings that should stop generation. Defaults to ["###"].
            temperature (float, optional): Temperature for the generation. Defaults to 1.0.
            model_kwargs (Dict, optional): Arguments to pass to the llm. Defaults to {}. Refer to the OpenAI python library documentation and OpenAI API documentation for more information.
        """
        self.max_tokens = max_tokens
        self.stop = stop
        self.model = model
        self.chat_template = None
        self.model_kwargs = model_kwargs
        self.temperature = temperature

    def query_model(self, messages: list, request_timeout: int = 600):
        chat = ChatOpenAI(
            model_name=self.model,  # type: ignore
            max_tokens=self.max_tokens,
            temperature=self.temperature,
            model_kwargs=self.model_kwargs,
            request_timeout=request_timeout,
        )
        completion = chat.invoke(messages, stop=self.stop)
        return completion

class ZeroShotNer(BaseNer):
    """Zero-shot NER model class."""

    def contextualize(
        self,
        entities: Dict[str, str],
        prompt_template: str = SYSTEM_TEMPLATE_EN,
        system_message_as_user_message: bool = False,
    ):
        """Method to ontextualize the zero-shot NER model. You don't need examples to contextualize this model.

        Args:
            entities (Dict[str, str]): Dict containing the entities to be recognized. The keys are the entity names and the values are the entity descriptions.
            prompt_template (str, optional): Prompt template to send the llm as the system message. Defaults to a prompt template for NER in English.
            system_message_as_user_message (bool, optional): If True, the system message will be sent as a user message. Defaults to False.
        """
        self.entities = entities
        if not system_message_as_user_message:
            system_template = SystemMessagePromptTemplate.from_template(prompt_template)
        else:
            system_template = HumanMessagePromptTemplate.from_template(prompt_template)
        self.system_message = system_template.format(
            entities=dict_to_enumeration(entities), entity_list=list(entities.keys())
        )
        self.chat_template = ChatPromptTemplate.from_messages(
            [
                self.system_message,
                HumanMessagePromptTemplate.from_template("{x}"),
            ]
        )

    def fit(self, *args, **kwargs):
        """Just a wrapper for the contextualize method. This method is here to be compatible with the sklearn API."""
        return self.contextualize(*args, **kwargs)

    def _predict(
        self, x: str, request_timeout: int
    ) -> AnnotatedDocument | AnnotatedDocumentWithException:
        messages = self.chat_template.format_messages(x=x)
        try:
            completion = self.query_model(messages, request_timeout)
        except Exception as e:
            logger.warning(
                f"The completion for the text '{x}' raised an exception: {e}"
            )
            return AnnotatedDocumentWithException(
                text=x, annotations=set(), exception=e
            )
        logger.debug(f"Completion: {completion}")
        annotated_document = inline_annotation_to_annotated_document(
            completion.content, list(self.entities.keys())
        )
        aligned_annotated_document = align_annotation(x, annotated_document)
        y = aligned_annotated_document
        return y

    def _predict_tokenized(self, x: List[str], request_timeout: int) -> Conll:
        detokenized_text = detokenizer(x)
        annotated_document = self._predict(detokenized_text, request_timeout)
        if isinstance(annotated_document, AnnotatedDocumentWithException):
            logger.warning(
                f"The completion for the text '{detokenized_text}' raised an exception: {annotated_document.exception}"
            )
        conll = annotated_document_to_conll(annotated_document)
        if not len(x) == len(conll):
            logger.warning(
                "The number of tokens and the number of conll tokens are different"
            )
        return conll

    def _predict_parallel(
        self, x: List[str], max_workers: int, progress_bar: bool, request_timeout: int
    ) -> List[AnnotatedDocument | AnnotatedDocumentWithException]:
        y = []
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            for annotated_document in tqdm(
                executor.map(lambda x: self._predict(x, request_timeout), x),
                disable=not progress_bar,
                unit=" example",
                total=len(x),
            ):
                y.append(annotated_document)
        return y

    def _predict_tokenized_parallel(
        self,
        x: List[List[str]],
        max_workers: int,
        progress_bar: bool,
        request_timeout: int,
    ) -> List[List[Conll]]:
        y = []
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            for conll in tqdm(
                executor.map(lambda x: self._predict_tokenized(x, request_timeout), x),
                disable=not progress_bar,
                unit=" example",
                total=len(x),
            ):
                y.append(conll)
        return y

    def _predict_serial(
        self, x: List[str], progress_bar: bool, request_timeout: int
    ) -> List[AnnotatedDocument | AnnotatedDocumentWithException]:
        y = []
        for text in tqdm(x, disable=not progress_bar, unit=" example"):
            annotated_document = self._predict(text, request_timeout)
            y.append(annotated_document)
        return y

    def _predict_tokenized_serial(
        self, x: List[List[str]], progress_bar: bool, request_timeout: int
    ) -> List[List[Conll]]:
        y = []
        for tokenized_text in tqdm(x, disable=not progress_bar, unit=" example"):
            conll = self._predict_tokenized(tokenized_text, request_timeout)
            y.append(conll)
        return y

    def predict(
        self,
        x: List[str],
        progress_bar: bool = True,
        max_workers: int = 1,
        request_timeout: int = 600,
    ) -> List[AnnotatedDocument | AnnotatedDocumentWithException]:
        """Method to perform NER on a list of strings.

        Args:
            x (List[str]): List of strings.
            progress_bar (bool, optional): If True, a progress bar will be displayed. Defaults to True.
            max_workers (int, optional): Number of workers to use for parallel processing. If -1, the number of workers will be equal to the number of CPU cores. Defaults to 1.
            request_timeout (int, optional): Timeout in seconds for the requests. Defaults to 600 seconds.

        Raises:
            NotContextualizedError: Error if the model is not contextualized before calling the predict method.
            ValueError: The input must be a list of strings.

        Returns:
            List[AnnotatedDocument | AnnotatedDocumentWithException]: List of AnnotatedDocument objects if there were no exceptions, a list of AnnotatedDocumentWithException objects if there were exceptions.
        """
        if self.chat_template is None:
            raise NotContextualizedError(
                "You must call the contextualize method before calling the predict method"
            )
        if not isinstance(x, list):
            raise ValueError("x must be a list")
        if isinstance(x[0], str):
            if max_workers == -1:
                y = self._predict_parallel(x, CPU_COUNT, progress_bar, request_timeout)
            elif max_workers == 1:
                y = self._predict_serial(x, progress_bar, request_timeout)
            elif max_workers > 1:
                y = self._predict_parallel(
                    x, max_workers, progress_bar, request_timeout
                )
            else:
                raise ValueError("max_workers must be greater than 0")
        else:
            raise ValueError(
                "x must be a list of strings, maybe you want to use predict_tokenized instead?"
            )
        return y

    def predict_tokenized(
        self,
        x: List[List[str]],
        progress_bar: bool = True,
        max_workers: int = 1,
        request_timeout: int = 600,
    ) -> List[List[Conll]]:
        """Method to perform NER on a list of tokenized documents.

        Args:
            x (List[List[str]]): List of lists of tokens.
            progress_bar (bool, optional): If True, a progress bar will be displayed. Defaults to True.
            max_workers (int, optional): Number of workers to use for parallel processing. If -1, the number of workers will be equal to the number of CPU cores. Defaults to 1.
            request_timeout (int, optional): Timeout in seconds for the requests. Defaults to 600 seconds.

        Returns:
            List[List[Conll]]: List of lists of tuples of (token, label).
        """
        if not isinstance(x, list):
            raise ValueError("x must be a list")
        if isinstance(x[0], list):
            if max_workers == -1:
                y = self._predict_tokenized_parallel(
                    x, CPU_COUNT, progress_bar, request_timeout
                )
            elif max_workers == 1:
                y = self._predict_tokenized_serial(x, progress_bar, request_timeout)
            elif max_workers > 1:
                y = self._predict_tokenized_parallel(
                    x, max_workers, progress_bar, request_timeout
                )
            else:
                raise ValueError("max_workers must be greater than 0")
        else:
            raise ValueError(
                "x must be a list of lists of tokens, maybe you want to use predict instead?"
            )
        return y

class FewShotNer(ZeroShotNer):
    def contextualize(
        self,
        entities: Dict[str, str],
        examples: List[AnnotatedDocument],
        prompt_template: str = SYSTEM_TEMPLATE_EN,
        system_message_as_user_message: bool = False,
    ):
        """Method to ontextualize the few-shot NER model. You need examples to contextualize this model.

        Args:
            entities (Dict[str, str]): Dict containing the entities to be recognized. The keys are the entity names and the values are the entity descriptions.
            examples (List[AnnotatedDocument]): List of AnnotatedDocument objects containing the annotated examples.
            prompt_template (str, optional): Prompt template to send the llm as the system message. Defaults to a prompt template for NER in English. Defaults to a prompt template for NER in English.
            system_message_as_user_message (bool, optional): If True, the system message will be sent as a user message. Defaults to False.
        """
        self.entities = entities
        if not system_message_as_user_message:
            system_template = SystemMessagePromptTemplate.from_template(prompt_template)
        else:
            system_template = HumanMessagePromptTemplate.from_template(prompt_template)
        self.system_message = system_template.format(
            entities=dict_to_enumeration(entities), entity_list=list(entities.keys())
        )
        example_template = ChatPromptTemplate.from_messages(
            [("human", "{input}"), ("ai", "{output}")]
        )
        few_shot_template = FewShotChatMessagePromptTemplate(
            examples=list(map(annotated_document_to_few_shot_example, examples)),
            example_prompt=example_template,
        )
        self.chat_template = ChatPromptTemplate.from_messages(
            [
                self.system_message,
                few_shot_template,
                HumanMessagePromptTemplate.from_template("{x}"),
            ]
        )


from itertools import chain

class GameState:
    def __init__(self, data):
        self.grid = data

    @classmethod
    def from_array(cls, grid):
        return cls(grid)

    @classmethod
    def empty(cls):
        grid = [[0, 0, 0, 0],
                [0, 0, 0, 0],
                [0, 0, 0, 0],
                [0, 0, 0, 0]]
        return cls(grid)

    def validate_sudoku_solution(self):

        def validate_rows(grid):
            for i in range(len(grid)):
                if sum(grid[i]) != sum(set(grid[i])):
                    return False
            return True

        def validate_columns(grid):
            t_grid = list(zip(*grid))
            return validate_rows(t_grid)

        def validate_squares(grid):
            for i in range(0, 4, 2):
                for j in range(0, 4, 2):
                    square = list(chain(*(row[j:j + 2] for row in grid[i:i + 2])))
                    if sum(square) != sum(set(square)):
                        return False
            return True

        rows_status = validate_rows(self.grid)
        columns_status = validate_columns(self.grid)
        squares_status = validate_squares(self.grid)

        valid = rows_status and columns_status and squares_status

        if valid:
            return 'Solution valid!'

        message = 'Failed: '
        if not rows_status:
            message += 'rows '
        if not columns_status:
            message += 'columns '
        if not squares_status:
            message += 'squares '

        return message


"""empty message

Revision ID: a99faa0cc155
Revises: 4a907061bba7
Create Date: 2021-04-23 21:15:51.931598

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'a99faa0cc155'
down_revision = '4a907061bba7'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('users', schema=None) as batch_op:
        batch_op.add_column(sa.Column('username', sa.String(length=255, collation='NOCASE'), nullable=False))
        batch_op.drop_constraint('uq_users_email', type_='unique')
        batch_op.create_unique_constraint(batch_op.f('uq_users_username'), ['username'])
        batch_op.drop_column('email')

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('users', schema=None) as batch_op:
        batch_op.add_column(sa.Column('email', sa.VARCHAR(length=255), nullable=False))
        batch_op.drop_constraint(batch_op.f('uq_users_username'), type_='unique')
        batch_op.create_unique_constraint('uq_users_email', ['email'])
        batch_op.drop_column('username')

    # ### end Alembic commands ###


#!/usr/bin/env python3
"""
Created on Wed Dec 25 2020

@author: beizishaozi
"""

import logging
import sys
import shutil
import re

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

import os
from libs.modules.BaseModule import BaseModule

logging.basicConfig(stream=sys.stdout, format="%(levelname)s: %(message)s", level=logging.INFO)
log = logging.getLogger(__name__)

class AppsGeyser(BaseModule):
    def doSigCheck(self):
        if self.host_os == "android":
            return self._find_main_activity('com.appsgeyser.multiTabApp.MainNavigationActivity')
        elif self.host_os == "ios":
            log.error("not support yet.")
            return False

        return False

    def doExtract(self, working_folder):
        extract_folder = self._format_working_folder(working_folder)
        if os.access(extract_folder, os.R_OK):
            shutil.rmtree(extract_folder)
        os.makedirs(extract_folder, exist_ok = True)
        tmp_folder = os.path.join(os.getcwd(), extract_folder, "tmp")
        os.makedirs(tmp_folder, exist_ok = True)
        self._apktool(tmp_folder)

        launch_path = ""
        for dirpath, dirnames, ifilenames in os.walk(tmp_folder):
            #audience_network.dex文件是框架自带的，经过apktool反编译之后在assets下会有一个文件夹，这部分信息没有必要提取
            if dirpath.find("audience_network") != -1:
                continue
            if dirpath.find("assets") != -1:   # store web resource
                for fs in ifilenames:
                    #这三个文件是框架自带的，没有必要提取
                    if fs == "audience_network.dex" or fs == "splash_screen.png" or fs == "user_custom_script.js":
                        continue
                    f = os.path.join(dirpath, fs)
                    matchObj = re.match(r'(.*)assets/(.*)', f, re.S)
                    newRP = matchObj.group(2)

                    tf = os.path.join(extract_folder, newRP)
                    if not os.access(os.path.dirname(tf), os.R_OK):
                        os.makedirs(os.path.dirname(tf))
                    with open(tf, "wb") as fwh:  #output the plain
                        # ugly coding
                        fp = open(os.path.join(dirpath, fs), "rb")
                        c = fp.read()
                        fp.close()
                        fwh.write(c)
                    fwh.close()
            elif dirpath.endswith("res/raw") != -1:
                for fs in ifilenames:
                    if fs != "configuration.xml":  #在configuration.xml里配置了起始页链接
                        continue
                    t = ET.ElementTree(file=os.path.join(dirpath, fs))
                    for elem in t.iter(tag='fullScreenMode'):
                        for subelem in elem.iter(tag='content'):
                            for linkelem in subelem.iter(tag='link'):
                                launch_path = linkelem.text

        self._dump_info(extract_folder, launch_path)
        # clean env
        shutil.rmtree(tmp_folder)
        return extract_folder, launch_path

def main():
    f = "./test_case/AppsGeyser/examplehtmlzip.apk"    #后续会将当前脚本路径与之相拼接，得到最终detect_file路径
    appsgeyser = AppsGeyser(f, "android")
    if appsgeyser.doSigCheck():
        logging.info("Andromo signature Match")

        extract_folder, launch_path = appsgeyser.doExtract("working_folder")
        log.info("{} is extracted to {}, the start page is {}".format(f, extract_folder, launch_path))

    return

if __name__ == "__main__":
    sys.exit(main())


import sys
from datetime import datetime

import tcpparser.environment as environment
import tcpparser.utils as utils

def is_blocked(ip: str) -> bool:
    """ Check if ip is already blocked """
    cmd = "{} {} {} {} {}".format("iptables", "-C INPUT", "-s", ip, "-j DROP")
    return environment.execute(cmd)

def block_ip(ip: str) -> None:
    """
    Blocks source ip
    e.g: iptables -I INPUT -s 12.34.56.78/32 -j DROP
    """
    # SHORTCUT: Ideally should check if the ip is not one of the ips on any
    # physic or virtual network device before blocking.
    cmd = "{} {} {} {}/32 {}".format("iptables", "-I INPUT", "-s", ip, "-j DROP")
    exe = cmd.split(' ')[0]

    if environment.exe_exists(exe):

        if not is_blocked(ip):
            return environment.execute(cmd)

    else:
        print(f"[ERROR]: Unable to execute: {exe}", file=sys.stderr)
        print(f"Please check if {exe} is installed and is in the right path")

def update_connections(cnx: dict, data: list, popfirst: bool) -> None:
    """ Update a dictionary (cnx) with the connections from the last minute """

    # dict key
    epoch = int(datetime.now().timestamp() * 1000)

    # After 1 minute, start removing the first entry
    # cnx is an ordered dictionary
    # FIFO
    if popfirst:
        cnx.pop(next(iter(cnx)))

    # Uses set to avoid duplication of values
    group_conn = set()

    for ldata in data:

        if ldata["DIRECTION"] == "->":
            conn = tuple((ldata["PEER"].split(':')[0], ldata["LOCAL"]))
            group_conn.add(conn)

    cnx[epoch] = group_conn

def check_portscan(cnx: dict, blocked_ips: list) -> None:
    """ Checks established connections from a source ip """

    peers = utils.organize_peer_connections(cnx)

    # Checks if a single peer address has more than 3 connection in the previous minute
    # and shows "Port scan detected"
    for host_ip, sockets in peers.items():
        fail = False

        # Check if IP wasn't already blocked and has more than 3 ports
        # open on host running tcpparser
        if len(sockets) > 3 and host_ip not in blocked_ips:
            blocked_ips.append(host_ip)
            now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            ports = []
            reason = "Port scan detected"

            for netaddr in sockets:
                ports.append(str(netaddr.split(':')[1]))

            local = str(netaddr.split(':')[0])
            print("{:>19}: {:>18}: {:>21} {} {:<21}".format(
                now, reason, host_ip, "->", local + " on ports " + ",".join(ports)
            ))

            # Blocks the source ip only once
            if not fail:
                fail = block_ip(host_ip)


import pickle
import socket
from datetime import timedelta, datetime
from dnslib import DNSError, DNSRecord

class Packet:
    def __init__(self, rr, create_time):
        self.resource_record = rr  # ресурсная запись
        self.create_time = create_time  # время создания записи

#  загркзка кэша из файла
def load_history():
    try:
        with open('data.pickle', 'rb') as f:
            database = pickle.load(f)
        print('history loaded')
    except:
        print('history not exist')
        return {}
    return database

#  дампинг кеша в файл
def save_history(data):
    try:
        with open('data.pickle', 'wb') as f:
            pickle.dump(data, f)
        print('dumping successful')
    except:
        print('dumping error')

#  проверка записи, истёк ttl или нет
def is_expired(packet):
    return datetime.now() - packet.create_time > timedelta(seconds=packet.resource_record.ttl)

#  очистка просроченного кеша
def clear_outdated_cash():
    cache_delta = 0
    for key, value in database.items():
        old_length = len(value)
        database[key] = set(packet for packet in value if not is_expired(packet))
        cache_delta += old_length - len(database[key])
    if cache_delta > 0:
        print(str(datetime.now()) + " - cleared " + str(cache_delta) + " resource records")

#  добавление/обновление записи в кеше
def add_record(rr, date_time):
    k = (str(rr.rname).lower(), rr.rtype)
    if k in database:
        database[k].add(Packet(rr, date_time))
    else:
        database[k] = {Packet(rr, date_time)}

#  извлечение ресурсных записей с ответом, записей с указанием на уполномоченный сервер, записей с дополнительной информацией
def add_records(dns_record):
    for r in dns_record.rr + dns_record.auth + dns_record.ar:
        date_time = datetime.now()
        add_record(r, date_time)
        print(str(date_time) + " - DNS record added.")

#  получение ответа из кеша, если он там есть
def get_response(dns_record):
    key = (str(dns_record.q.qname).lower(), dns_record.q.qtype)
    if key in database and database[key]:
        reply = dns_record.reply()
        reply.rr = [p.resource_record for p in database[key]]
        return reply

#  отправить результат юзеру
def send_response(response, addr):
    sock.connect(addr)
    sock.sendall(response)
    sock.close()

#  основной цикл
def work_loop():
    global sock

    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.bind(("", 40000))

    try:
        while True:
            data, addr = sock.recvfrom(2048)

            if database:
                clear_outdated_cash()

            try:
                dns_record = DNSRecord.parse(data)
            except DNSError:
                print('getting incorrect query, packet ignored')
                continue

            add_records(dns_record)
            if not dns_record.header.qr:
                response = get_response(dns_record)
                try:
                    if response:
                        send_response(response.pack(), addr)
                    else:
                        resp = dns_record.send("dns.yandex.ru")
                        add_records(DNSRecord.parse(resp))
                        send_response(resp, addr)
                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                    sock.bind(("", 40000))
                    print(str(datetime.now()) + " - response send")
                except (OSError, DNSError):
                    print(str(datetime.now()) + " - transmission error")
    except:
        print('server error')

def main():
    print('server started')

    global database

    database = load_history()

    try:
        work_loop()
    finally:
        if database:
            save_history(database)
        print('server stop')

if __name__ == '__main__':
    main()


import telebot
import yaml
import sys

from telebot import types

import db_access

config = yaml.safe_load(open("conf.yml"))

bot = telebot.TeleBot(config["token"])

match_dict = {}
pred_dict = {}

class Prediction:
    def __init__(self, uid):
        self.userid = uid
        self.matchid = None
        self.win = None
        self.goal = None
        self.total = None

class Match:
    def __init__(self, teamname1, teamname2):
        self.team1 = teamname1
        self.team2 = teamname2

#################### FUNCTIONS TO READ DATA ####################

# Function to handle the /start command:
# After sending this command, user gets a message how to use bot
# chat_id is added to the database (kind of user registration)
@bot.message_handler(commands=['start'])
def send_start(message):
    try:
        chat_id = message.chat.id
        bot.send_message(chat_id,
                         'Welcome to Euro 2016 prediction contest! ' +
                         'Type /help for help, ' +
                         '/rules for rules, or ' +
                         '/predict to make your predictions')
        db_access.add_user(chat_id)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

# Function to handle the /help command:
# User gets the list of all available commands
@bot.message_handler(commands=['help'])
def send_help(message):
    try:
        chat_id = message.chat.id
        bot.send_message(chat_id,
                         '/start View start message\n' +
                         '/help View this message\n' +
                         '/predict Make prediction\n' +
                         '/setname Set your name\n' +
                         '/setgroup Set your group\n' +
                         '/rules Show rules\n' +
                         '/today See what happens today\n' +
                         '/ranking See the ranking of your groups\n' +
                         '/current See the predictions for current match')
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

# Function to handle the /rules command:
# User gets a short description of how the points are assigned
@bot.message_handler(commands=['rules'])
def send_rules(message):
    try:
        chat_id = message.chat.id
        bot.send_message(chat_id,
                         'You need to answer three questions about each match:\n' +
                         '1. Who will win the game or will there be a draw?\n' +
                         '2. What will be the difference of team goals? ' +
                         '(in case you predicted draw: how many goals will each team score?)\n' +
                         '3. What will be total of the match? (Over or Under 2.5 golas?)\n' +
                         'You may get 3 points for each correct answer.' +
                         ' If all 3 answers are correct, you get an additional point.')
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

# Function to handle the /today command:
# User gets a list of matches that will be played today
# The user also gets his predictions for each match
@bot.message_handler(commands=['today'])
def send_today_info(message):
    try:
        chat_id = message.chat.id
        today_matches = db_access.today_matches()
        if (len(today_matches) == 0):
            bot.send_message(chat_id, 'No matches today!')
        today_matches_ids = [c['match_no'] for c in today_matches]
        user_profile = db_access.get_user(chat_id)
        info_text = 'Matches and your predictions for today:\n'
        if (user_profile == 0):
            return (0)
        else:
            user_predictions = user_profile['predictions']
            today_user_prediction_ids = [c['match_no'] for c in user_predictions if c['match_no'] in today_matches_ids]
            # TODO: make map instead of for-loop:
            for match in today_matches:
                info_text = info_text + match['team_1'] + " vs. " + match['team_2'] + ': '
                if match['match_no'] in today_user_prediction_ids:
                    prediction = [c for c in user_predictions if c['match_no'] == match['match_no']][0]
                    if (prediction['winner'] == 0):
                        info_text = info_text + 'Draw '
                        info_text = info_text + str(prediction['goals']) + ':' + str(prediction['goals']) + ', '
                    else:
                        if (prediction['winner'] == 1):
                            info_text = info_text + match['team_1']
                        elif (prediction['winner'] == 2):
                            info_text = info_text + match['team_2']
                        info_text = info_text + ' by ' + str(prediction['goals']) + ' goals, '
                    if (prediction['total'] == 0):
                        info_text = info_text + '<2.5'
                    elif (prediction['total'] == 1):
                        info_text = info_text + '>2.5'
                else:
                    info_text = info_text + 'No prediction (yet)'
                info_text = info_text + '\n'
            info_text = info_text + 'Click /predict to make Predictions!'
            bot.send_message(chat_id, info_text)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

# Function to handle the /current command
# User gets predictions of members of all his groups
# for the current matches.
# Current match is defined as not finished and not active match
@bot.message_handler(commands=['current'])
def send_current_match_info(message):
    try:
        chat_id = message.chat.id
        user_profile = db_access.get_user(chat_id)
        current_matches = db_access.current_matches()
        if(len(current_matches) == 0):
            bot.send_message(chat_id, 'There are no matches now!')
        else:
            for match in current_matches:
                predictions_for_match = db_access.get_predictions_match(match, user_profile)
                string_to_send = match['team_1'] + ' vs. ' + match['team_2'] + ':\n'
                for prediction_for_match in predictions_for_match:
                    if ('name' in prediction_for_match):
                        string_to_send = string_to_send + prediction_for_match['name'] + ': '
                    if ('predictions' in prediction_for_match):
                        prediction_l = prediction_for_match['predictions']
                        prediction = prediction_l[0]
                        if (prediction['winner'] == 0):
                            string_to_send = string_to_send + 'Draw ' + str(prediction['goals']) + ':' + str(prediction['goals']) + ', '
                        else:
                            if (prediction['winner'] == 1):
                                string_to_send = string_to_send + match['team_1']
                            elif (prediction['winner'] == 2):
                                string_to_send = string_to_send + match['team_2']
                            string_to_send = string_to_send + ' by ' + str(prediction['goals']) + ' goal(s), '
                        if (prediction['total'] == 0):
                            string_to_send = string_to_send + '<2.5\n'
                        elif (prediction['total'] == 1):
                            string_to_send = string_to_send + '>2.5\n'
                    else:
                        string_to_send = string_to_send + 'No prediction\n'
                bot.send_message(chat_id, string_to_send)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

# Help function to create a printable string from group name and ranking list
def ranking_to_str(group, ranking):
    result = group + '\n'
    # TODO: add formatting (probably, with additional function)
    for rank in ranking:
        if ('name' in rank):
            result = result + rank['name'] + '   ' + str(rank['total_points']) + '\n'
        else:
            result = result +  '   ' + str(rank['total_points']) + '\n'
    return result

# Function to handle the /ranking command
# The user gets one message per group (where he is a member)
# This message contains sorted list of group members with their points
# In case the user is not a member of any group, he may request a group,
# that he wants to get ranking for
# (this request is processed in the next function).
@bot.message_handler(commands=['ranking'])
def get_ranking(message):
    try:
        chat_id = message.chat.id
        groups = db_access.get_groups(chat_id)
        if (len(groups) == 0):
            msg = bot.reply_to(message,
                               'You are not a member of any groups! ' +
                               'Please enter the name of the group you want to get ranking!')
            bot.register_next_step_handler(msg, process_ranking)
        else:
            user_groups = groups['groups']
            for group in user_groups:
                ranking = db_access.get_ranking(group)
                ranking_sorted = sorted(ranking, key=lambda k: k['total_points'], reverse = True)
                ranking_str = ranking_to_str(group, ranking_sorted)
                msg = bot.reply_to(message, ranking_str)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

# Function called if a user is not a member of any group:
# Ranking of chosen group is shown.
def process_ranking(message):
    try:
        chat_id = message.chat.id
        group = message.text
        ranking = db_access.get_ranking(group)
        ranking_sorted = sorted(ranking, key=lambda k: k['total_points'], reverse = True)
        ranking_str = ranking_to_str(group, ranking_sorted)
        msg = bot.reply_to(message, ranking_str)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

#################### FUNCTIONS TO WRITE DATA ####################

# Function to handle the /setname command
# User can set his name
@bot.message_handler(commands=['setname'])
def set_name(message):
    try:
        msg = bot.reply_to(message, 'What is your name?')
        bot.register_next_step_handler(msg, process_name)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

def process_name(message):
    try:
        chat_id = message.chat.id
        name = message.text
        db_access.add_name(chat_id, name)
        bot.send_message(chat_id, "Nice to meet you, " + name + "!")
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

@bot.message_handler(commands=['setgroup'])
def set_group(message):
    try:
        msg = bot.reply_to(message, "Type the name of the group you want to be added to:")
        bot.register_next_step_handler(msg, process_group)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

def process_group(message):
    try:
        chat_id = message.chat.id
        group_name = message.text
        db_access.add_group(chat_id, group_name)
        bot.send_message(chat_id, "You were added to the group " + group_name + "!")
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

@bot.message_handler(commands=['predict'])
def send_predict(message):
    try:
        chat_id = message.chat.id
        pred = Prediction(chat_id)
        pred_dict[chat_id] = pred
        next_match = db_access.get_next_match(chat_id)
        if(next_match == 0):
            bot.send_message(chat_id, "Sorry! There are no matches in our database you can currently predict!")
        else:
            pred.matchid = [c['match_no'] for c in next_match][0]
            team1 = [c['team_1'] for c in next_match][0]
            team2 = [c['team_2'] for c in next_match][0]
            match = Match(team1, team2)
            match_dict[chat_id] = match
            stage = [c['type'] for c in next_match][0]
            quote = [c['quote'] for c in next_match][0]
            if (stage == 'Group'):
                stage = stage + ' ' + [c['group'] for c in next_match][0]
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            team1button = types.KeyboardButton(match.team1)
            team2button = types.KeyboardButton(match.team2)
            drawbutton = types.KeyboardButton("Draw")
            markup.add(team1button, team2button, drawbutton)
            msg = bot.reply_to(message, stage + ': ' + match.team1 + ' vs. ' + match.team2 + '(Quote: ' + str(quote) + ')\n Who will win the game?', reply_markup=markup)
            bot.register_next_step_handler(msg, process_win_step)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')
        print("Unexpected error:", sys.exc_info()[0])

def process_win_step(message):
    try:
        chat_id = message.chat.id
        match = match_dict[chat_id]
        print("Prediction started for the match: " + match.team1 + " vs. " + match.team2)
        pred = pred_dict[chat_id]
        if (message.text == match.team1):
            pred.win = 1
        elif (message.text == match.team2):
            pred.win = 2
        elif (message.text == u'Draw'):
            pred.win = 0
        else:
            bot.send_message(chat_id, "Error!")
        print("Chat id: " + str(chat_id) + "; Prediction: " + message.text + ' (' + str(pred.win) + ')')
        if (message.text == u'Draw'):
            msg = bot.reply_to(message, "How many goals will each team score?")
            bot.register_next_step_handler(msg, process_goal_step)
        else:
            msg = bot.reply_to(message, "What will be the goal difference?")
            bot.register_next_step_handler(msg, process_goal_step)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

def process_goal_step(message):
    try:
        chat_id = message.chat.id
        pred = pred_dict[chat_id]
        print("Chat id: " + str(chat_id) + "; Prediction: " + message.text)
        pred.goal = int(message.text)
        underbutton = types.KeyboardButton('< 2.5')
        overbutton = types.KeyboardButton('> 2.5')
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add(underbutton, overbutton)
        msg = bot.reply_to(message, 'How many goals will the teams score?', reply_markup=markup)
        bot.register_next_step_handler(msg, process_total_step)
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')
        print("Unexpected error:", sys.exc_info()[0])

def process_total_step(message):
    try:
        chat_id = message.chat.id
        pred = pred_dict[chat_id]
        print('Chat id: ' + str(chat_id) + '; Prediction: ' + message.text)
        if (message.text == '< 2.5'):
            pred.total = 0
        elif (message.text == '> 2.5'):
            pred.total = 1
        else:
            msg = bot.send_message(chat_id, "Error!")
        db_access.add_prediction(pred.userid, pred.matchid, pred.win, pred.goal, pred.total)
        bot.send_message(chat_id, "Your prediction for this game is saved! Click /predict to make the next prediction! Click /today to see today's matches and your predictions!")
    except Exception as e:
        bot.reply_to(message, 'Something went wrong! Please, contact the provider of this bot!')

@bot.message_handler(func=lambda message: True)
def echo_all(message):
    return (0)
    # bot.reply_to(message, "Unfortunately we do not support free text.
    # If you see this message, something went wrong
    # (or you just typed some text the bot can not understand).")

bot.polling()


import json
import logging
import psycopg2
import redis
import requests
import time
from contextlib import contextmanager

LOG = logging.getLogger()
APP_PORT = 8080
REDIS_PORT = 8081
PSQL_PORT = 8082

class DBConnector():
    def __init__(self, host='localhost', port=PSQL_PORT, db='postgres', user='postgres'):
        self.connect = psycopg2.connect(
            host=host,
            port=port,
            dbname=db,
            user=user
        )

    def fetch_all_votes(self):
        """
        Fetches all rows from the votes table and returns a tuple
        of tuples in the format ((id1, vote1), (id2, vote2), ...)
        """
        with self.connect.cursor() as curs:
            curs.execute(
                """
                SELECT * FROM votes;
                """
            )
            return curs.fetchall()

    def fetch_vote_by_id(self, voter_id):
        """
        Fetches the vote value of the specified `voter_id`.  Raises
        exception on multiple matches.  Returns None for non-match.
        """
        with self.connect.cursor() as curs:
            curs.execute(
                """
                SELECT vote from votes where id = %(id)s;
                """,
                {'id': voter_id}
            )
            results = curs.fetchall()

            if len(results) == 0:
                return None
            assert len(results) == 1, "Votes for id {} not equal to 1: {}".format(voter_id, results)
            return results[0][0]

    @contextmanager
    def expect_vote_updated(self, voter_id, timeout=60):
        """
        Context manager that waits 60 seconds for the db to update
        before return.
        """
        old = self.fetch_vote_by_id(voter_id)

        try:
            yield self
        finally:
            timeout = time.time() + 60
            while time.time() < timeout:
                new = self.fetch_vote_by_id(voter_id)
                if old != new:
                    return
                LOG.warning("Expected vote value not updated")
                time.sleep(1)

class RedisConnector():
    def __init__(self, host='localhost', port=REDIS_PORT):
        self.client = redis.Redis(
            host=host,
            port=port
        )

    def push_vote(self, vote, voter_id):
        """
        Pushes a vote in the back of the queue
        """
        ret = self.client.rpush(
            'votes',
            json.dumps({
                "vote": vote,
                "voter_id": voter_id
            })
        )
        assert ret == 1

    def insert_vote(self, vote, voter_id):
        """
        Inserts a vote in the front of the queue
        """
        ret = self.client.lpush(
            'votes',
            json.dumps({
                "vote": vote,
                "voter_id": voter_id
            })
        )
        assert ret == 1

class APIConnector():
    def __init__(self, host='localhost', port=APP_PORT):
        self.host = host
        self.port = port
        self.session = requests.Session()

    @property
    def voter_id(self):
        """
        Returns the voter_id from the session cookie.  Opens a
        new session if not yet opened
        """
        if 'voter_id' not in self.session.cookies:
            LOG.warning("No voter_id cookie")
            self._init_cookies()
        return self.session.cookies.get('voter_id')

    @property
    def url(self):
        """
        Returns the formated url
        """
        return "http://{}:{}/".format(self.host, self.port)

    def _init_cookies(self):
        """
        GET the web service for initializing the cookie
        """
        LOG.debug("Getting voter_id cookie")
        self.session.get(self.url)

    def _post(self, data, *args, **kwargs):
        """
        Wrapper session.post.  Raises exception on HTTP
        error codes.
        """
        resp = self.session.post(
            self.url,
            data,
            *args,
            **kwargs
        )
        resp.raise_for_status()
        return resp

    def post_vote(self, vote, voter_id=None):
        """
        POST a vote to the web service
        """
        if voter_id:
            return self._post(
                data={'vote': vote},
                cookies={'voter_id', voter_id}
            )
        return self._post(
            data={'vote': vote}
        )


from django.urls import include, path
from .views import BoardViewSet, CategoryViewSet, TaskViewSet, UserView
from rest_framework import routers

router = routers.DefaultRouter()
router.register(r'boards', BoardViewSet, basename='boards')
router.register(r'boards/(?P<id>[\w-]+)/tasks', TaskViewSet, basename='boardtasks')
router.register(r'boards/(?P<id>[\w-]+)/categories', CategoryViewSet, basename='boardcategories')

urlpatterns = [
    path('users/', UserView.as_view()),
    path('boards/<int:id>/users/', UserView.as_view()),
    path('', include(router.urls))
]


from typing import List
from os.path import isfile
from collections import OrderedDict
import logging

from scapy.layers.dot11 import RadioTap, Dot11, Dot11FCS
from scapy.packet import Packet, Raw
from scapy.utils import rdpcap
import pcapy

from netzob.Common.NetzobException import NetzobImportException
from netzob.Common.Utils.SortedTypedList import SortedTypedList
from netzob.Import.PCAPImporter.PCAPImporter import PCAPImporter
from netzob.Model.Vocabulary.Messages.AbstractMessage import AbstractMessage
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
from netzob.Model.Vocabulary.Messages.L2NetworkMessage import L2NetworkMessage
from netzob.Model.Vocabulary.Messages.L4NetworkMessage import L4NetworkMessage

from nemere.validation.messageParser import ParsingConstants

class BaseLoader(object):
    """
    Wrapper for loading messages from memory as specimens.
    Prepares and represents messages to support further analysis.
    Especially useful for on-the-fly creation of test cases.
    """

    def __init__(self, l5msgs, l1msgs=None, baselayer=None):
        """
        Load messages from memory. Base class for other loaders, e. g. loading from PCAP file.

        :param l5msgs:
        :param l1msgs:
        :param baselayer: Optionally, set the base layer explicitly.
        """
        if not hasattr(self, 'pcapFileName'):
            self.pcapFileName = 'from-memory'
        self.messagePool = OrderedDict()  # type: OrderedDict[AbstractMessage, RawMessage]
        """maps the message representations for Netzob and tshark
        dict of { application layer of messages L5Messages : RawMessage }"""
        if not l1msgs:
            l1msgs = l5msgs
        for k, m in zip(l5msgs, l1msgs):
            self.messagePool[k] = m
            # TODO replace the above quickfix not to read the file a second time (should we?)
            # probably we could use msgs = ParsedMessage.parseMultiple(l1msgs); for m in msgs:
            # ... append(RawMessage(m.protocolbytes))
        self._baselayer = baselayer

    def getBaseLayerOfPCAP(self):
        """
        see ParsingConstants.LINKTYPES

        :return: Determine lowest encapsulating layer of PCAP.
        """
        if self._baselayer is not None:
            return self._baselayer

        try:
            # looking at just one message should reveal lowest encapsulating layer of the whole PCAP
            al5msg = next(iter(self.messagePool.keys()))
        except StopIteration:
            raise ValueError('No message could be imported. See previous errors for more details.')
        if isinstance(al5msg, L2NetworkMessage):
            if al5msg.l2Protocol == 'Ethernet':
                return ParsingConstants.LINKTYPES['ETHERNET']
            # for some reason, its not possible to access the class variable "name", so instantiate a dummy object
            elif al5msg.l2Protocol == Dot11().name or al5msg.l2Protocol == Dot11FCS().name:
                return ParsingConstants.LINKTYPES['IEEE802_11']  # 802.11
            elif al5msg.l2Protocol == 'None':  # no ethernet
                if al5msg.l3Protocol == 'IP':
                    return ParsingConstants.LINKTYPES['RAW_IP']  # IP
                else:
                    raise NotImplementedError("Linktype on layer 3 unknown. Protocol is {}".format(al5msg.l3Protocol))
            else:
                raise NotImplementedError("Linktype on layer 2 unknown. Protocol is {}".format(al5msg.l2Protocol))
        else:
            return ParsingConstants.LINKTYPES['undecoded']  # non-decoded raw trace without link type information

    def __repr__(self):
        return type(self).__name__ + ": " + self.pcapFileName + f" on layer {self.getBaseLayerOfPCAP()}"

    @property
    def maximumMessageLength(self):
        """
        :return: The maximum message length in bytes of the relevant network layer without its encapsulation for
            the messages in this specimen's pool.
        """
        return max(len(line.data) for line in self.messagePool.keys())

    @property
    def cumulatedMessageLength(self):
        """
        :return: The sum of all message lengths in bytes of the relevant network layer without its encapsulation for
            the messages in this specimen's pool. I. e., the cumulated size of all payload in the trace.
        """
        return sum(len(line.data) for line in self.messagePool.keys())

class SpecimenLoader(BaseLoader):
    """
    Wrapper for loading messages from a PCAP as specimens.
    Prepares and represents messages to support further analysis.
    """

    def __init__(self, pcap: str, layer:int=-1, relativeToIP:bool=False):
        """
        Load the messages from the PCAP file of the given name.

        >>> from nemere.utils.loader import SpecimenLoader
        >>> sl = SpecimenLoader('../input/hide/random-100-continuous.pcap', layer=0, relativeToIP=True)
        >>> firstmessage = list(sl.messagePool.items())[0]
        >>> print(firstmessage[0].data.hex())  # the whole message
        450000780001000040007c837f0000017f000001da362a9b658bcc70d1a8323979e8fc0affda6f9fca9de16a3b051303601280f2c249abdd05f85ce98aebaf67626c07b4698a3c87abb95af87abf108735bac6ffd8823e80ac498622e7347852e0f2e7a8759911782b00a74b5744109b9fe59f5d311252a1
        >>> print(firstmessage[1].data.hex())  # only the payload of the target layer
        ffffffffffff0000000000000800450000780001000040007c837f0000017f000001da362a9b658bcc70d1a8323979e8fc0affda6f9fca9de16a3b051303601280f2c249abdd05f85ce98aebaf67626c07b4698a3c87abb95af87abf108735bac6ffd8823e80ac498622e7347852e0f2e7a8759911782b00a74b5744109b9fe59f5d311252a1

        :param pcap: PCAP input file name.
        :param layer: The protocol layer to extract. If not set or negative, use the top layer.
        :param relativeToIP: If True, extract the given layer relative to the IP layer.

        """
        if not isfile(pcap):
            raise FileNotFoundError('File not found:', pcap)
        self.pcapFileName = pcap
        self.layer = layer
        self.relativeToIP = relativeToIP
        absLayer = 2 + layer if relativeToIP else layer

        # prevent Netzob from producing debug output in certain cases.
        logging.getLogger().setLevel(30)

        try:
            if layer < 0:
                # read messages at layer 5 for the Netzob inference
                l5msgs = PCAPImporter.readFile(pcap, importLayer=5).values()  # type: List[L4NetworkMessage]
            else:
                # read messages at given layer for the Netzob inference
                l5msgs = PCAPImporter.readFile(pcap, importLayer=absLayer).values()  # type: List[AbstractMessage]
            # read messages as raw for tshark input
            l1msgs = PCAPImporter.readFile(pcap, importLayer=1).values()  # type: List[RawMessage]
        except (NetzobImportException, pcapy.PcapError):
            importer = ScaPyCAPimporter(self.pcapFileName, absLayer)
            l5msgs = importer.messages
            l1msgs = importer.rawMessages
        super().__init__(l5msgs, l1msgs)

    # The int value of some pcapy datalink denotations is different from the tcpdump ones: https://www.tcpdump.org/linktypes.html
    # http://vpnb.leipzig.freifunk.net:8004/srv2/lede/lede-20171116/build_dir/target-mips_24kc_musl/python-pcapy-0.11.1/pcapy.html#idp8777598240
    pcapyDatalinkTranslation = {
        pcapy.DLT_RAW: ParsingConstants.LINKTYPES['RAW_IP']
    }
    """Translates pcapy linktype values to tcpdump ones."""

    def getBaseLayerOfPCAP(self):
        pcap = pcapy.open_offline(self.pcapFileName)
        dl = pcap.datalink()
        # Translates pcapy linktype values to tcpdump ones if in dict, otherwise the value is used unchanged
        return dl if dl not in SpecimenLoader.pcapyDatalinkTranslation else SpecimenLoader.pcapyDatalinkTranslation[dl]

class ScaPyCAPimporter(object):
    def __init__(self, pcapfilename, importLayer=5):
        # l5msgs = PCAPImporter.readFile(pcap, importLayer=absLayer).values()  # type: List[AbstractMessage]
        self.importLayer = importLayer
        self.packets = rdpcap(pcapfilename)
        self._messages = SortedTypedList(AbstractMessage)
        self._rawmessages = SortedTypedList(AbstractMessage)

        for pkt in self.packets:  # type: Packet
            self.packetHandler(pkt)

    @property
    def messages(self):
        return self._messages.values()

    @property
    def rawMessages(self):
        return self._rawmessages.values()

    def packetHandler(self, packet: Packet):
        epoch = packet.time
        l1Payload = bytes(packet)
        if len(l1Payload) == 0:
            return
        # Build the RawMessage
        rawMessage = RawMessage(l1Payload, epoch, source=None, destination=None)

        if isinstance(packet, RadioTap):
            # lift layer to Dot11 if there is a RadioTap dummy frame
            packet = packet.payload
        if self.importLayer == 2:
            (l2Proto, l2SrcAddr, l2DstAddr, l2Payload) = self.__decodeLayer2(packet)
            if len(l2Payload) == 0:
                return
            # Build the L2NetworkMessage
            l2Message = L2NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr)
            self._messages.add(l2Message)
            self._rawmessages.add(rawMessage)
        else:
            # Use Netzob's PCAPImporter if layer 2 is not WLAN
            raise NetzobImportException("PCAP", "Unsupported import layer. Currently only handles layer 2.",
                                        PCAPImporter.INVALID_LAYER2)

    def __decodeLayer2(self, packet: Packet):
        """Internal method that parses the specified header and extracts
        layer2 related proprieties."""
        l2Proto = packet.name
        if isinstance(packet, Raw):
            print("Ignoring undecoded packet with values:", bytes(packet).hex())
            return l2Proto, None, None, ""
        if isinstance(packet, Dot11):
            l2DstAddr = packet.fields['addr1']  # receiver address, alt: packet.fields['addr3'] destination address
            l2SrcAddr = packet.fields['addr2']  # transmitter address, alt: packet.fields['addr4'] source address
        else:
            raise NetzobImportException("NEMERE_PCAP", "Unsupported layer 2 protocol " + l2Proto,
                                        PCAPImporter.INVALID_LAYER2)
        l2Payload = bytes(packet.payload)
        return l2Proto, l2SrcAddr, l2DstAddr, l2Payload


from pydantic import BaseModel
from typing import List, Optional

# File
class FileBase(BaseModel):
    title: str
    description: str | None = None

class File(FileBase):
    id: int
    description: str | None = None
    owner_id: int

    class Config:
        orm_mode = True

class FileSchema(BaseModel):
    own_files: list[File] = []
    shared_files: list[File] = []

# Token
class Token(BaseModel):
    access_token: str
    token_type: str

class TokenData(BaseModel):
    username: Optional[str] = None
    scopes: List[str] = []

# User
class UserBase(BaseModel):
    username: str

class UserCreate(UserBase):
    password: str

class User(UserBase):
    id: int

    class Config:
        orm_mode = True

class UserSchema(User):
    email: Optional[str] = None
    full_name: Optional[str] = None
    disabled: Optional[bool] = None
    is_admin: bool
    files: list[File] = []

class Perm(BaseModel):
    id: int
    user_id: int
    file_id: int
    view: bool
    rename: bool
    delete: bool

    class Config:
        orm_mode = True

# Sessions
class SessionData(BaseModel):
    token: str


import argparse
import asyncio
import importlib
import json
import os
import re

import cv2 as cv
import numpy as np
from PySide2.QtCore import *
import websockets

class MMRWSData():
  def __init__(self):
    self.requestType = ''
    self.dataType = ''
    self.data = {}

  def load_from_bytes(self, data):
    byteArray = QByteArray(data)
    inStream = QDataStream(byteArray)
    inStream.setVersion(QDataStream.Qt_5_9)

    self.requestType = inStream.readQString()
    self.dataType = inStream.readQString()
    self.data = inStream.readQVariant()

  def to_bytes(self):
    byteArray = QByteArray()
    outStream = QDataStream(byteArray, QIODevice.WriteOnly)
    outStream.setVersion(QDataStream.Qt_5_9)

    outStream.writeQString(self.requestType)
    outStream.writeQString(self.dataType)
    outStream.writeQVariant(self.data)

    return byteArray.data()

# parser
parser = argparse.ArgumentParser()

parser.add_argument('--config_path', help='Path of the config file', type=str)

args = parser.parse_args()

# config
config = json.load(open(args.config_path, 'r'))

# model
model_module = importlib.import_module(config['model']['name'])
model = model_module.create_model(config['model']['config'])

# modalities
modality_identifier_list = []
modality_info_list = []
modality_identifier_to_name_dict = {}
for modality_info in config['modalities']:
  if (not 'identifier' in modality_info):
    modality_info['identifier'] = ''
  if (len(modality_info['identifier']) <= 0):
    identifier = input('- input identifier for %s: ' % (modality_info['name']))
    modality_info['identifier'] = identifier

  modality_identifier_to_name_dict[modality_info['identifier']] = modality_info['name']

  modality_identifier_list.append(modality_info['identifier'])
  modality_info_list.append(modality_info)

# websocket functions
def wsPrepare():
  model.prepare()

def wsStart():
  model.start()

def wsRecognize(data):
  new_data = {}
  for (identifier, item) in data.items():
    new_item = {}
    for subitem in item:
      if (subitem['type'] == 'value'):
        new_item[subitem['name']] = subitem['value']
      elif (subitem['type'] == 'byte_array'):
        new_item[subitem['name']] = subitem['byte_array'].data()
    new_data[modality_identifier_to_name_dict[identifier]] = new_item
  return model.recognize(new_data)

def wsStop():
  model.stop()

def wsFinalize():
  model.finalize()

# run websocket
async def run():
  print('- connecting...')
  async with websockets.connect(config['server']['url'], max_size=(2**32), read_limit=(2**32), write_limit=(2**32), max_queue=(2**8)) as websocket:
    wsData = MMRWSData()
    wsData.requestType = 'register'
    wsData.dataType = 'request'
    wsData.data['name'] = config['model']['name']
    wsData.data['identifier'] = config['model']['identifier']
    wsData.data['modalities'] = modality_identifier_list
    wsData.data['fps'] = config['model']['fps']

    await websocket.send(wsData.to_bytes())
    receivedData = await websocket.recv()
    print('- connected')

    while True:
      try:
        receivedData = await websocket.recv()
        wsData = MMRWSData()
        wsData.load_from_bytes(receivedData)

        print(wsData.requestType)

        if (wsData.requestType == 'prepare'):
          wsPrepare()
        elif (wsData.requestType == 'start'):
          wsStart()
        elif (wsData.requestType == 'recognize'):
          res = wsRecognize(wsData.data)

          resWsData = MMRWSData()
          resWsData.requestType = 'recognize'
          resWsData.dataType = 'response'
          resWsData.data = res
          await websocket.send(resWsData.to_bytes())

        elif (wsData.requestType == 'stop'):
          wsStop()
        elif (wsData.requestType == 'finalize'):
          wsFinalize()
          break
      except Exception as e:
        print(e)
        break

asyncio.get_event_loop().run_until_complete(run())

# finalize


import numpy as np
import pandas_datareader as pdr
import datetime as dt
import pandas as pd
import plotly.graph_objects as go
import calendar
import requests

symbol='BTCUSD' #symbol to be traded
tick_interval = '1' #candle in minutes

now = dt.datetime.utcnow()
unixtime = calendar.timegm(now.utctimetuple())
since = unixtime
start = str(since - 60 * 60 * int(tick_interval))
url = 'https://api.bybit.com/v2/public/kline/list?symbol='+symbol+'&interval='+tick_interval+'&from='+str(start)
data = requests.get(url).json()
D = pd.DataFrame(data['result'])

a = 1
c = 10

high_low = D['high'].astype(float) - D['low'].astype(float)
high_close = np.abs(D['high'].astype(float) - D['close'].astype(float).shift())
low_close = np.abs(D['low'].astype(float) - D['close'].astype(float).shift())
ranges = pd.concat([high_low, high_close, low_close], axis=1)
true_range = np.max(ranges, axis=1)

atr_indicator = true_range.rolling(c).sum()/c
nLoss = atr_indicator * a

print("############ ATR ############")
print(atr_indicator)
print("#############################")


import numpy as np
import tensorflow as tf
import sys
import math
import matplotlib.pyplot as plt

import amplitf.interface as atfi
import tfa.rootio as atfr
import tfa.plotting as atfp

def create_weights_biases(n_input, layers, sigma=1.0, n_output=1):
    """
    Create arrays of weights and vectors of biases for the multilayer perceptron
    if a given configuration (with a single output neuron).
      n_input : number of input neurons
      layers  : list of numbers of neurons in the hidden layers
    output :
      weights, biases
    """
    n_hidden = [n_input] + layers
    weights = []
    biases = []
    for i in range(len(n_hidden) - 1):
        weights += [
            tf.Variable(
                sigma * np.random.normal(size=[n_hidden[i], n_hidden[i + 1]]),
                dtype=atfi.fptype(),
            )
        ]
        biases += [
            tf.Variable(
                sigma * np.random.normal(size=[n_hidden[i + 1]]), dtype=atfi.fptype()
            )
        ]
    weights += [
        tf.Variable(
            sigma * np.random.normal(size=[n_hidden[-1], n_output]), dtype=atfi.fptype()
        )
    ]
    biases += [
        tf.Variable(sigma * np.random.normal(size=[n_output]), dtype=atfi.fptype())
    ]
    return (weights, biases)

def init_weights_biases(init):
    """
    Initialise variable weights and biases from numpy array
    """
    init_weights = init[0]
    init_biases = init[1]
    weights = []
    biases = []
    for i in range(len(init_weights) - 1):
        weights += [tf.Variable(init_weights[i], dtype=atfi.fptype())]
        biases += [tf.Variable(init_biases[i], dtype=atfi.fptype())]
    weights += [tf.Variable(init_weights[-1], dtype=atfi.fptype())]
    biases += [tf.Variable(init_biases[-1], dtype=atfi.fptype())]
    return (weights, biases)

def init_fixed_weights_biases(init):
    """
    Initialise constant weights and biases from numpy array
    """
    init_weights = init[0]
    init_biases = init[1]
    weights = []
    biases = []
    for i in range(len(init_weights) - 1):
        weights += [tf.constant(init_weights[i], dtype=atfi.fptype())]
        biases += [tf.constant(init_biases[i], dtype=atfi.fptype())]
    weights += [tf.constant(init_weights[-1], dtype=atfi.fptype())]
    biases += [tf.constant(init_biases[-1], dtype=atfi.fptype())]
    return (weights, biases)

def normalise(array, ranges):
    v = [
        (array[:, i] - atfi.const(ranges[i][0])) / (atfi.const(ranges[i][1]) - atfi.const(ranges[i][0]))
        for i in range(len(ranges))
    ]
    return tf.stack(v, axis=1)

@tf.function
def multilayer_perceptron(x, ranges, weights, biases, multiple=False):
    """
    Multilayer perceptron with fully connected layers defined by matrices of weights and biases.
    Use sigmoid function as activation.
    """
    layer = normalise(x, ranges)
    for i in range(len(weights)):
        layer = tf.nn.sigmoid(tf.add(tf.matmul(layer, weights[i]), biases[i]))
    if multiple:
        return layer
    else:
        return layer[:, 0]

def l2_regularisation(weights):
    """
    L2 regularisation term for a list of weight matrices
    """
    penalty = 0.0
    for w in weights:
        penalty += tf.reduce_sum(tf.square(w))
    return penalty

def estimate_density(
    phsp,
    data,
    ranges,
    labels,
    weight=None,
    transform = None,
    transform_ranges = None,
    learning_rate=0.001,
    training_epochs=100000,
    norm_size=1000000,
    print_step=50,
    display_step=500,
    weight_penalty=1.,
    n_hidden=[32, 8],
    initfile="init.npy",
    outfile="train",
    seed=1,
    fig = None,
    axes = None,
    units = None,
    model = None,
    regularisation = None,
):

    n_input = len(ranges)

    bins = n_input*[ 50 ]

    atfi.set_seed(seed)

    if model is None :
        try:
            init_w = np.load(initfile, allow_pickle = True)
        except:
            init_w = None

        if isinstance(init_w, np.ndarray):
            print("Loading saved weights")
            (weights, biases) = init_weights_biases(init_w[2:])
        else:
            print("Creating random weights")
            (weights, biases) = create_weights_biases(n_input, n_hidden)

        if not transform_ranges : transform_ranges = ranges

        def fitmodel(x):
            if transform : x2 = transform(x)
            else : x2 = x
            # to make sure PDF is always strictly positive
            return multilayer_perceptron(x2, transform_ranges, weights, biases) + 1e-20

        parameters = [weights, biases]
    else :
        fitmodel, parameters = model

    @tf.function
    def unbinned_nll(pdf, integral) :
        return -tf.reduce_sum(atfi.log(pdf/integral))

    @tf.function
    def unbinned_weighted_nll(pdf, integral, w) :
        return -tf.reduce_sum(atfi.log(pdf/integral)*w)

    @tf.function
    def integral(pdf) :
        return tf.reduce_mean(pdf)

    opt = tf.keras.optimizers.Adam(learning_rate=learning_rate)

    data_inside = phsp.inside(data)
    data_sample = atfi.cast_real(data[data_inside])
    if weight is not None :
        weight_sample = weight[data_inside]
    else :
        weight_sample = None

    if isinstance(norm_size, int) :
        norm_sample = phsp.uniform_sample(norm_size)
    else :
        norm_sample = norm_size
    print("Normalisation sample size = ", len(norm_sample))
    print(norm_sample)
    print("Data sample size = ", len(data_sample))
    print(data_sample)

    if weight_sample is None :
        @tf.function
        def raw_nll() :
            return unbinned_nll(fitmodel(data_sample), integral(fitmodel(norm_sample)))
    else :
        @tf.function
        def raw_nll() :
            return unbinned_weighted_nll(fitmodel(data_sample), integral(fitmodel(norm_sample)), weight_sample)

    if model is None :
        # Define loss and optimizer
        if regularisation is None :
            @tf.function
            def nll() :
                return raw_nll() + l2_regularisation(weights)*weight_penalty
        else :
            @tf.function
            def nll() :
                return raw_nll() + regularisation(weights)
    else :
        @tf.function
        def nll() :
            return raw_nll()

    # Training cycle
    best_cost = 1e10

    display = atfp.MultidimDisplay(
        data_sample,
        norm_sample,
        bins,
        ranges,
        labels,
        fig,
        axes,
        units = units,
        dataweights = weight_sample
    )
    plt.ion()
    plt.show()
    plt.pause(0.1)

    for epoch in range(training_epochs):

        if epoch % display_step == 0 and fig :
            w = fitmodel(norm_sample)
            display.draw(w)
            plt.tight_layout(pad=0., w_pad=0., h_pad=0.)
            plt.draw()
            plt.pause(0.1)
            plt.savefig(outfile + ".pdf")

        if epoch % print_step == 0 :
            c = nll()
            s = "Epoch %d, cost %.9f" % (epoch+1, c)
            print(s)
            if c < best_cost:
                best_cost = c
                w = fitmodel(norm_sample)
                scale = 1./np.mean(w)
                if model is None :
                    weights = [w.numpy() for w in parameters[0]]
                    biases =  [b.numpy() for b in parameters[1]]
                    np.save(outfile, [ scale, transform_ranges, weights, biases] )
                else :
                    np.save(outfile, [ scale, transform_ranges] + [p.numpy() for p in parameters() ] )
                f = open(outfile + ".txt", "w")
                f.write(s + "\n")
                f.close()

        opt.minimize(nll, parameters)

    print("Optimization Finished!")

def estimate_density_old(
    phsp,
    data,
    ranges,
    labels,
    weight=None,
    transform=None,
    transform_ranges=None,
    learning_rate=0.001,
    training_epochs=100000,
    norm_size=1000000,
    print_step=50,
    display_step=500,
    weight_penalty=1.0,
    n_hidden=[32, 8],
    initfile="init.npy",
    outfile="train",
    seed=1,
    fig=None,
    axes=None,
):

    tf.compat.v1.disable_eager_execution()

    n_input = len(ranges)

    bins = n_input * [50]

    #tf.compat.v1.set_random_seed(seed)
    #np.random.seed(seed + 12345)

    try:
        init_w = np.load(initfile, allow_pickle=True)
    except:
        init_w = None

    if isinstance(init_w, np.ndarray):
        print("Loading saved weights")
        (weights, biases) = init_weights_biases(init_w)
    else:
        print("Creating random weights")
        (weights, biases) = create_weights_biases(n_input, n_hidden)

    data_ph = tf.compat.v1.placeholder(atfi.fptype(), shape=(None, None), name="data")
    norm_ph = tf.compat.v1.placeholder(atfi.fptype(), shape=(None, None), name="norm")

    if not transform_ranges:
        transform_ranges = ranges

    def model(x):
        if transform:
            x2 = transform(x)
        else:
            x2 = x
        # to make sure PDF is always strictly positive
        return multilayer_perceptron(x2, transform_ranges, weights, biases) + 1e-20

    data_model = model(data_ph)
    norm_model = model(norm_ph)

    def unbinned_nll(pdf, integral):
        return -tf.reduce_sum(atfi.log(pdf / integral))

    def integral(pdf):
        return tf.reduce_mean(pdf)

    # Define loss and optimizer
    nll = (
        unbinned_nll(data_model, integral(norm_model))
        + l2_regularisation(weights) * weight_penalty
    )

    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)
    train_op = optimizer.minimize(nll)

    # Initializing the variables
    init = tf.compat.v1.global_variables_initializer()

    with tf.compat.v1.Session() as sess:
        sess.run(init)

        data_sample = sess.run(phsp.filter(data_ph), feed_dict={data_ph: data})

        norm_sample = sess.run(phsp.uniform_sample(norm_size))
        print("Normalisation sample size = ", len(norm_sample))
        print(norm_sample[:,0])
        print(norm_sample[:,1])
        print(norm_sample[:,2])
        print(norm_sample[:,3])
        print("Data sample size = ", len(data_sample))
        print(data_sample)

        # Training cycle
        best_cost = 1e10

        display = atfp.MultidimDisplay(
            data_sample, norm_sample, bins, ranges, labels, fig, axes
        )
        plt.ion()
        plt.show()
        plt.pause(0.1)

        for epoch in range(training_epochs):

            _, c = sess.run(
                [train_op, nll], feed_dict={data_ph: data_sample, norm_ph: norm_sample}
            )

            if epoch % display_step == 0 and fig:
                w = sess.run(norm_model, feed_dict={norm_ph: norm_sample})
                display.draw(w)
                plt.tight_layout(pad=1.0, w_pad=1.0, h_pad=1.0)
                plt.draw()
                plt.pause(0.1)
                plt.savefig(outfile + ".pdf")

            if epoch % print_step == 0:
                s = "Epoch %d, cost %.9f" % (epoch + 1, c)
                print(s)
                if c < best_cost:
                    best_cost = c
                    w = sess.run(norm_model, feed_dict={norm_ph: norm_sample})
                    scale = 1.0 / np.mean(w)
                    np.save(
                        outfile, [scale, transform_ranges] + sess.run([weights, biases])
                    )
                    f = open(outfile + ".txt", "w")
                    f.write(s + "\n")
                    f.close()

        print("Optimization Finished!")


#! /usr/bin/env python
# coding=utf-8

import tensorflow as tf
import os
import time
import datetime
import data_input_helper as data_helpers
from tensor_textcnn import TextCNN
# from text_cnn import TextCNN
import math
from tensorflow.contrib import learn
import pandas as pd
import warnings
warnings.filterwarnings("ignore")
import numpy as np

import jieba
import jieba.posseg as pseg
from tensor_birnn import TextBiRNN
# from text_birnn import TextBiRNN
from tensor_textdnn import TextDNN
from tensor_textrcnn import TextRCNN
from tensor_textrnn import TextRNN
from tensor_textfast import TextFast

from utils import get_vocabulary_size, fit_in_vocabulary, zero_pad, batch_generator

# Parameters
# ==================================================
# Data loading params
model_type = "clf" #"the type of model ,classify or regression(defalut=clf)"
using_nn_type = "text_cnn" #the type of network(default:textcnn)
dev_sample_percentage = 0.1 # "Percentage of the training data to use for validation"
train_data_file =  "./data/cutclean_label_corpus10000.txt" #"Data source for the positive data."
train_label_data_file = "" #"Data source for the label data."
w2v_file =  "./data/vectors.bin"  #"w2v_file path"

# Model Hyperparameters
embedding_dim = 128 #"Dimensionality of character embedding (default: 128)"
filter_sizes = "2,3,4"  # "Comma-separated filter sizes (default: '3,4,5')"
num_filters = 128  #"Number of filters per filter size (default: 128)"
dropout_keep_prob = 0.5  #"Dropout keep probability (default: 0.5)"
l2_reg_lambda = 0.0  #"L2 regularization lambda (default: 0.0)"
hidden_size = 128  #"Number of hiddern layer units(defalut:128)"
hidden_layers  = 2  # "Number of hidden layers(default:2)"
rnn_size = 300  #"num of units rnn_size(default:3)"
num_rnn_layers  = 3  #"number of rnn layers(default:3)"

# Training parameters
batch_size = 64  #"Batch Size (default: 64)"
num_epochs  = 10  # "Number of training epochs (default: 200)"
evaluate_every = 50  #"Evaluate model on dev set after this many steps (default: 100)"
checkpoint_every = 100  #"Save model after this many steps (default: 100)"
num_checkpoints = 5  ##"Number of checkpoints to store (default: 5)"

# Misc Parameters
allow_soft_placement = True ## "Allow device soft device placement"
log_device_placement = False  ###"Log placement of ops on devices"

# FLAGS = tf.flags.FLAGS
# FLAGS.flag_values_dict()
# print("\nParameters:")
# for attr, value in sorted(FLAGS.flag_values_dict().items()):
#     print("{}={}".format(attr.upper(), value))
# print("")

#
def load_data(w2v_model=None):
    print("laoding data")
    x_text, y = data_helpers.load_data_and_labels(train_data_file)
    max_document_length = max([len(x.split(" ")) for x in x_text])
    if (w2v_model == None):
        vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
        x = np.array(list(vocab_processor.fit_transform(x_text)))
        vocab_size = len(vocab_processor.vocabulary_)
    else:
        x = data_helpers.get_text_idx(x_text, w2v_model.vocab_hash, max_document_length)
        vocab_size = len(w2v_model.vocab_hash)
        print('use w2v .bin')

    np.random.seed(10)
    shuffle_indices = np.random.permutation(np.arange(len(y)))
    x_shuffled = x[shuffle_indices]
    y_shuffled = y[shuffle_indices]

    dev_sample_index = -1 * int(dev_sample_percentage * float(len(y)))
    x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
    y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]

    return x_train, x_dev, y_train, y_dev, vocab_size

# #####################--------------载入数据----

x_train1, x_dev1, y_train1, y_dev1, vocab_size1 = load_data(None)
#############--------定义nn类型
nn_type = "text_rnn"

############-----------------------载入imdb数据
from keras.datasets import imdb
from keras.preprocessing import sequence
from keras.utils import to_categorical
max_features = 10000
maxlen = 500
# batch_size = 32
print("load data")
(input_train,y_train),(input_test,y_test) = imdb.load_data(num_words=max_features)
print(len(input_train),"train_sequence")
print(len(input_test),"test sequence")

print("padding _squence")
input_train = sequence.pad_sequences(input_train,maxlen=maxlen)
input_test = sequence.pad_sequences(input_test,maxlen=maxlen)
print(input_train.shape,"input_trian_shape")
print(input_test.shape,"input_test_shape")
######----------y_train转化为one_hot
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
##########----------字典的大小-----
vocab_size = np.max([np.max(input_train[i]) for i in range(input_train.shape[0])])+1
print("vocab_size,--字典大小   ",vocab_size)

#####-------------------d带入训练
x_train = input_train.copy()

x_dev = input_test.copy()
y_dev = y_test.copy()

def train():
    # Training
    # ==================================================
    # x_train, x_dev, y_train, y_dev ,vocab_size= load_data(w2v_model)
    with tf.Graph().as_default():
        session_conf = tf.ConfigProto(
            allow_soft_placement=allow_soft_placement,
            log_device_placement=log_device_placement)
        sess = tf.Session(config=session_conf)
        with sess.as_default():
            if (nn_type == "text_cnn"):
                nn = TextCNN(
                    model_type=model_type,
                    sequence_length=x_train.shape[1],
                    num_classes=y_train.shape[1],
                    vocab_size=vocab_size,
                    embedding_size=embedding_dim,
                    filter_sizes=list(map(int, filter_sizes.split(","))),
                    num_filters=num_filters,
                    l2_reg_lambda=l2_reg_lambda
                )
            elif nn_type == "text_birnn":
                nn = TextBiRNN(
                    model_type=model_type,
                    sequence_length=x_train.shape[1],
                    num_classes=y_train.shape[1],
                    vocab_size=vocab_size,
                    embedding_size=embedding_dim,
                    rnn_size=128,
                    num_layers=3,
                    l2_reg_lambda=l2_reg_lambda
                )
            elif nn_type == "text_rnn":
                nn = TextRNN(
                    model_type=model_type,
                    sequence_length=x_train.shape[1],
                    num_classes=y_train.shape[1],
                    vocab_size=vocab_size,
                    embedding_size=embedding_dim,
                    rnn_size=128,
                    num_layers=3,
                    l2_reg_lambda=l2_reg_lambda
                )
            elif nn_type == "text_rcnn":
                nn = TextBiRNN(
                    model_type=model_type,
                    sequence_length=x_train.shape[1],
                    num_classes=y_train.shape[1],
                    vocab_size=vocab_size,
                    embedding_size=embedding_dim,
                    rnn_size=128,
                    num_layers=3,
                    l2_reg_lambda=l2_reg_lambda
                )
            elif nn_type == "text_dnn":
                nn = TextDNN(
                    model_type=model_type,
                    sequence_length=x_train.shape[1],
                    num_classes=y_train.shape[1],
                    vocab_size=vocab_size,
                    embedding_size=embedding_dim,
                    hidden_layes=2,
                    hidden_size=128,
                    l2_reg_lambda=l2_reg_lambda
                )
            elif nn_type == "text_fasttext":
                nn = TextFast(
                    model_type=model_type,
                    sequence_length=x_train.shape[1],
                    num_classes=y_train.shape[1],
                    vocab_size=vocab_size,
                    embedding_size=embedding_dim,
                    l2_reg_lambda=l2_reg_lambda
                )

            # Define Training procedure
            global_step = tf.Variable(0, name="global_step", trainable=False)
            optimizer = tf.train.AdamOptimizer(1e-3)
            grads_and_vars = optimizer.compute_gradients(nn.loss)
            train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)

            # Initialize all variables
            sess.run(tf.global_variables_initializer())

            def train_step(x_batch, y_batch):
                """
                A single training step
                """
                feed_dict = {
                    nn.input_x: x_batch,
                    nn.input_y: y_batch,
                    nn.dropout_keep_prob: dropout_keep_prob
                }
                _, step,loss, accuracy = sess.run(
                    [train_op, global_step, nn.loss, nn.accuracy],
                    feed_dict)

                time_str = datetime.datetime.now().isoformat()
                print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
                # print w[:2],idx[:2]
                # train_summary_writer.add_summary(summaries, step)

            def dev_step(x_batch, y_batch, writer=None):
                """
                Evaluates model on a dev set
                """
                feed_dict = {
                    nn.input_x: x_batch,
                    nn.input_y: y_batch,
                    nn.dropout_keep_prob: 1.0
                }

                step,loss, accuracy = sess.run(
                    [global_step, nn.loss, nn.accuracy],
                    feed_dict)
                #
                time_str = datetime.datetime.now().isoformat()
                print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
                # if writer:
                #     writer.add_summary(summaries, step)

            # Generate batches
            batches = data_helpers.batch_iter(
                list(zip(x_train, y_train)),batch_size, num_epochs)

            def dev_test():
                batches_dev = data_helpers.batch_iter(list(zip(x_dev, y_dev)), batch_size, 1)
                for batch_dev in batches_dev:
                    x_batch_dev, y_batch_dev = zip(*batch_dev)
                    dev_step(x_batch_dev, y_batch_dev, writer=None)

            # Training loop. For each batch...
            for batch in batches:
                x_batch, y_batch = zip(*batch)
                train_step(x_batch, y_batch)
                #
                # current_step = tf.train.global_step(sess, global_step)

if __name__ == "__main__":
    # w2v_wr = data_helpers.w2v_wrapper(FLAGS.w2v_file)
    train()
    attrs = []
    values = []
    # for attr, value in sorted(FLAGS.__flags.items()):
    #     attrs += [attr]
    #     values += [value]
    # info = pd.DataFrame()
    # info["attr"] = attrs
    # info["value"] = values
    # info.to_csv(os.path.curdir + '/config.csv', index=False)


from datetime import date
hoje = date.today()
print('''{:-^50}
O formato padrão será como o exemplo: {}27/06/1991{}
{:-^50}'''.format(' CADASTRO DATA DE NASCIMENTO ', '\033[1:33m', '\033[m', ''))
totmaior = 0
totmenor = 0
for pess in range(1, 8):
    n = input('Favor digitar a {}ª data de nascimento: '.format(pess)).strip().replace('/', ' ').split()
    n = [int(n[2]), int(n[1]), int(n[0])]
    n = date(n[0], n[1], n[2])
    i = (hoje - n).days/365.25
    if i >= 18:
        totmaior += 1
    else:
        totmenor += 1
print('Ao todo tivemos {} pessoas maiores de idade.\nTambém tivemos {} pessoas menores de idade.'.format(totmaior, totmenor))

Trainning Scripts/Curso em Vídeo/Mundo 2/2 - Estrutura de Repetição FOR/ex054 - Grupo de Maioridade.py


from PyQt5.QtWidgets import QWidget, QInputDialog, QFontDialog, QFileDialog
import os

class Dialogs(QWidget):
    def nameAsk(self):
        name, ok = QInputDialog.getText(self, 'Input Name',
                                        'Your name please? ')
        if ok:
            self.tedit.setText('Hello ' + str(name) +
                               '. Did you sleep well?')

    def fontDialog(self):
        font, ok = QFontDialog.getFont()
        if ok:
            self.tedit.setFont(font)

    def saveDialog(self):
        filename = QFileDialog.getSaveFileName(self, 'Save File',
                                               os.getenv('HOME'))[0]
        f = open(filename, 'w')
        tedittext = self.tedit.toPlainText()
        f.write(tedittext)
        f.close()

    def openDialog(self):
        filename = QFileDialog.getOpenFileName(self, 'Open File',
                                               os.getenv('HOME'))[0]
        self.setWindowTitle('Laze Text Editor - ' + filename)
        f = open(filename, 'ra')
        self.tedit.setText(f.read())
        f.close()


import pytest

from applications.choices import ApplicationStatusChoice
from tests.factories import ApplicationFactory, ApplicationRoundFactory

# Applied to all tests
pytestmark = [
    pytest.mark.django_db,
    pytest.mark.usefixtures("_disable_elasticsearch"),
]

def test_reset_application_allocation():
    # given:
    # - There is a single handled application in an application round in allocation
    application_round = ApplicationRoundFactory.create_in_status_in_allocation()
    application = ApplicationFactory.create_in_status_handled(application_round=application_round)

    # when:
    # - User tries to reset allocation on the application
    application.actions.reset_application_allocation()

    # then:
    # - The application is in allocation again
    application.refresh_from_db()
    assert application.status == ApplicationStatusChoice.IN_ALLOCATION

def test_reset_application_round_allocation():
    # given:
    # - There is a single handled application in an application round in allocation
    application_round = ApplicationRoundFactory.create_in_status_in_allocation()
    application_1 = ApplicationFactory.create_in_status_handled(application_round=application_round)
    application_2 = ApplicationFactory.create_in_status_handled(application_round=application_round)
    application_3 = ApplicationFactory.create_in_status_handled(application_round=application_round)

    # when:
    # - User tries to reset allocation on the application
    application_round.actions.reset_application_round_allocation()

    # then:
    # - The application is in allocation again
    application_1.refresh_from_db()
    assert application_1.status == ApplicationStatusChoice.IN_ALLOCATION
    application_2.refresh_from_db()
    assert application_2.status == ApplicationStatusChoice.IN_ALLOCATION
    application_3.refresh_from_db()
    assert application_3.status == ApplicationStatusChoice.IN_ALLOCATION