from .util import graph_preprocessor, decoder_normalizer, combine_std
from torch.utils.data import Dataset
from .noise import gen_noise
from .graph import Graph
import numpy as np
import torch
import h5py

def collate_fn(batch):
    lengths = torch.tensor([x[0].n_nodes for x in batch])
    lengths = torch.cat([torch.tensor([0]), lengths[:-1]], dim=0)
    offsets = torch.cumsum(lengths, dim=0)

    nodes = []
    edges = []
    senders = []
    receivers = []

    for i, pair in enumerate(batch):
        graph = pair[0]
        nodes.append(graph.nodes)
        edges.append(graph.edges)
        senders.append(graph.senders + offsets[i])
        receivers.append(graph.receivers + offsets[i])

    graph = Graph(torch.cat(nodes, dim=0))
    graph.edges = torch.cat(edges, dim=0)
    graph.senders = torch.cat(senders, dim=0)
    graph.receivers = torch.cat(receivers, dim=0)

    # at inference time (batch_size = 1)
    graph.attrs = batch[0][0].attrs
    graph.pos = batch[0][0].pos
    graph.types = batch[0][0].types

    gt = [x[1] for x in batch]
    gt = torch.cat(gt, dim=0)

    return (graph, gt)

class SimulationDataset(Dataset):
    def __init__(self, file, group, vel_seq, noise_std, only_first=False):
        self.dataset = h5py.File(file, 'r', swmr=True, libver='latest')[group]
        # used for validation dataset
        self.only_first = only_first
        self.noise_std = noise_std
        self.rollouts = len(self.dataset['positions'].keys())
        self.vel_seq = vel_seq

    def __len__(self):
        self.len = self.rollouts * (self.dataset['positions'].get('0').shape[0] - self.vel_seq)
        return self.len

    def __getitem__(self, idx):
        num = str(idx // (self.dataset['positions']['0'].shape[0] - self.vel_seq))
        begin = np.random.randint(low=0, high=(self.dataset['positions']['0'].shape[0] - self.vel_seq - 2))

        if self.only_first:
            begin = 0

        particle_types = self.dataset['particle_types'].get(num)
        arr = np.zeros(particle_types.shape, dtype='double')
        particle_types.read_direct(arr)
        types = torch.tensor(arr).unsqueeze(1).float()

        positions = self.dataset['positions'].get(num)
        arr = np.zeros((self.vel_seq + 1, positions.shape[1], positions.shape[2]), dtype='double')
        positions.read_direct(arr, np.s_[begin: begin + self.vel_seq + 1])
        rollout = torch.tensor(arr).float()

        subseq = rollout[:self.vel_seq]
        noise = gen_noise(subseq, self.noise_std)
        subseq += noise

        previous_vel = subseq[-1] - subseq[-2]
        next_vel = (rollout[self.vel_seq] + noise[-1]) - subseq[-1]

        gt = next_vel - previous_vel
        attrs = self.dataset.attrs

        acc_std = torch.tensor(combine_std(attrs['acc_std'], self.noise_std))
        amean = torch.tensor(attrs['acc_mean'])
        gt = decoder_normalizer(gt, amean, acc_std)

        graph = graph_preprocessor(subseq, attrs, types)
        return [graph, gt.float()]


from unittest import TestCase

from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler

class UserAgentMiddlewareTest(TestCase):
    def get_spider_and_mw(self, default_useragent):
        crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
        spider = crawler._create_spider("foo")
        return spider, UserAgentMiddleware.from_crawler(crawler)

    def test_default_agent(self):
        spider, mw = self.get_spider_and_mw("default_useragent")
        req = Request("http://scrapytest.org/")
        assert mw.process_request(req, spider) is None
        self.assertEqual(req.headers["User-Agent"], b"default_useragent")

    def test_remove_agent(self):
        # settings USER_AGENT to None should remove the user agent
        spider, mw = self.get_spider_and_mw("default_useragent")
        spider.user_agent = None
        mw.spider_opened(spider)
        req = Request("http://scrapytest.org/")
        assert mw.process_request(req, spider) is None
        assert req.headers.get("User-Agent") is None

    def test_spider_agent(self):
        spider, mw = self.get_spider_and_mw("default_useragent")
        spider.user_agent = "spider_useragent"
        mw.spider_opened(spider)
        req = Request("http://scrapytest.org/")
        assert mw.process_request(req, spider) is None
        self.assertEqual(req.headers["User-Agent"], b"spider_useragent")

    def test_header_agent(self):
        spider, mw = self.get_spider_and_mw("default_useragent")
        spider.user_agent = "spider_useragent"
        mw.spider_opened(spider)
        req = Request(
            "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
        )
        assert mw.process_request(req, spider) is None
        self.assertEqual(req.headers["User-Agent"], b"header_useragent")

    def test_no_agent(self):
        spider, mw = self.get_spider_and_mw(None)
        spider.user_agent = None
        mw.spider_opened(spider)
        req = Request("http://scrapytest.org/")
        assert mw.process_request(req, spider) is None
        assert "User-Agent" not in req.headers


# Databricks notebook source
# DBTITLE 1,Web Scraping from Hindenburg Research
from bs4 import BeautifulSoup
import requests

url = "https://hindenburgresearch.com/page/{}/"
page_num = 1
page = requests.get(url.format(page_num))

# COMMAND ----------

soup = BeautifulSoup(page.text, 'html')
print(soup.prettify())

# COMMAND ----------

headline = soup.find('h1').text
print(headline)
print(type(headline))

# COMMAND ----------

titles_arr = []

next_text = soup.find("a", class_="next page-numbers")

while next_text is not None:
    #titles = soup.find_all(['h2','span'], class_ = ['post-title','entry-title-primary'])
    titles = soup.find_all(['h2'], class_ = ['post-title'])
    for title in titles:
        print(title.text)
        titles_arr.append(title.text)
    page_num+=1
    page = requests.get(url.format(page_num))
    soup = BeautifulSoup(page.text, 'html')
    next_text = soup.find("a", class_="next page-numbers")

# COMMAND ----------

#Last page that does not have next button
if next_text is None:
    titles = soup.find_all(['h2'], class_ = ['post-title'])
    for title in titles:
        print(title.text)
        titles_arr.append(title.text)

# COMMAND ----------

print(page_num)
print(url.format(page_num))

# COMMAND ----------

print(len(titles_arr))

# COMMAND ----------

for title_text in titles_arr:
    print(title_text)


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patheffects as path_effects

def plotConf(df_confusion,titulo,savefile):
  plt.ioff()
  fig = plt.figure()
  plt.clf()
  ax = fig.add_subplot(111)
  ax.set_aspect(1)
  res = ax.imshow(np.array(df_confusion), cmap=plt.cm.jet,
                  interpolation='nearest')

  width, height = np.array(df_confusion).shape

  for x in xrange(width):
      for y in xrange(height):
          ax.annotate(str(df_confusion[x][y]), xy=(y, x),
                      horizontalalignment='center',
                      verticalalignment='center', color='white',
                      fontweight='bold',
                      path_effects=[path_effects.withSimplePatchShadow(alpha=0.5)])

  plt.yticks( np.arange(height) )
  plt.xticks( np.arange(width) )
  plt.ylabel('Valor real')
  plt.xlabel('Valor predicho')
  cb = fig.colorbar(res)
  plt.title(titulo,fontsize=16)

  plt.savefig(savefile)
  return


import re
from datetime import datetime, timedelta
from functools import cache, wraps
from typing import Iterable

from cymple.builder import NodeAfterMergeAvailable, NodeAvailable, QueryBuilder

from ...model import (
    MatchStrategy,
    Node,
    Relationship,
    RelationshipIdentityShape,
    RelationshipWithNodes,
    TimeToLiveConfiguration,
)
from ...schema.schema import GraphObjectType
from ..query_executor import OperationOnNodeIdentity, OperationOnRelationshipIdentity
from .query import Query, QueryBatch

PROPERTIES_PARAM_NAME = "properties"
ADDITIONAL_LABELS_PARAM_NAME = "additional_labels"
GENERIC_NODE_REF_NAME = "node"
FROM_NODE_REF_NAME = "from_node"
TO_NODE_REF_NAME = "to_node"
RELATIONSHIP_REF_NAME = "rel"
PARAMETER_CORRECTION_REGEX = re.compile(r"\"(params.__\w+)\"")
DELETE_NODE_QUERY = "MATCH (n) WHERE id(n) = id DETACH DELETE n"
DELETE_REL_QUERY = "MATCH ()-[r]->() WHERE id(r) = id DELETE r"

def correct_parameters(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        query = f(*args, **kwargs)
        return PARAMETER_CORRECTION_REGEX.sub(r"\1", query)

    return wrapper

def generate_prefixed_param_name(property_name: str, prefix: str) -> str:
    return f"__{prefix}_{property_name}"

def generate_properties_set_with_prefix(properties: Iterable[str], prefix: str):
    return {
        prop: f"params.{generate_prefixed_param_name(prop, prefix)}"
        for prop in properties
    }

def generate_where_set_with_prefix(properties: frozenset, prefix: str):
    return {
        f"{prefix}.{prop}": f"params.{generate_prefixed_param_name(prop, prefix)}"
        for prop in properties
    }

@cache
def _match_node(
    node_operation: OperationOnNodeIdentity, name=GENERIC_NODE_REF_NAME
) -> NodeAvailable:
    op = "=~" if node_operation.match_strategy == MatchStrategy.FUZZY else "="
    identity = node_operation.node_identity
    props = generate_where_set_with_prefix(identity.keys, name)
    return (
        QueryBuilder()
        .match()
        .node(labels=identity.type, ref_name=name)
        .where_multiple(props, comparison_operator=op)
    )

@cache
def _merge_node(
    node_operation: OperationOnNodeIdentity, name=GENERIC_NODE_REF_NAME
) -> NodeAfterMergeAvailable:
    properties = generate_properties_set_with_prefix(
        node_operation.node_identity.keys, name
    )
    return (
        QueryBuilder()
        .merge()
        .node(
            labels=node_operation.node_identity.type,
            ref_name=name,
            properties=properties,
        )
    )

@cache
def _merge_relationship(rel_identity: RelationshipIdentityShape):
    keys = generate_properties_set_with_prefix(rel_identity.keys, RELATIONSHIP_REF_NAME)
    match_rel_query = (
        QueryBuilder()
        .match_optional()
        .node(ref_name=FROM_NODE_REF_NAME)
        .related_to(
            ref_name=RELATIONSHIP_REF_NAME,
            properties=keys,
            label=rel_identity.type,
        )
        .node(ref_name=TO_NODE_REF_NAME)
    )
    create_rel_query = str(match_rel_query).replace("OPTIONAL MATCH ", "CREATE")
    set_properties_query = f"SET {RELATIONSHIP_REF_NAME} += params.{generate_prefixed_param_name(PROPERTIES_PARAM_NAME, RELATIONSHIP_REF_NAME)}"

    return f"""
    {match_rel_query}
    FOREACH (x IN CASE WHEN {RELATIONSHIP_REF_NAME} IS NULL THEN [1] ELSE [] END |
        {create_rel_query} {set_properties_query})
    FOREACH (i in CASE WHEN {RELATIONSHIP_REF_NAME} IS NOT NULL THEN [1] ELSE [] END |
        {set_properties_query})
    """

class Neo4jIngestQueryBuilder:
    @cache
    @correct_parameters
    def generate_update_node_operation_query_statement(
        self,
        operation: OperationOnNodeIdentity,
    ) -> str:
        """Generate a query to update a node in the database given a node type and a match strategy."""

        if operation.match_strategy == MatchStrategy.EAGER:
            query = str(_merge_node(operation))
        else:
            query = str(_match_node(operation))

        result = f"{query} SET {GENERIC_NODE_REF_NAME} += params.{generate_prefixed_param_name(PROPERTIES_PARAM_NAME, GENERIC_NODE_REF_NAME)}"
        if operation.node_identity.additional_types:
            result += f" WITH {GENERIC_NODE_REF_NAME}, params CALL apoc.create.addLabels({GENERIC_NODE_REF_NAME}, params.{generate_prefixed_param_name(ADDITIONAL_LABELS_PARAM_NAME, GENERIC_NODE_REF_NAME)}) yield node RETURN true"
        return result

    def generate_update_node_operation_params(self, node: Node) -> dict:
        """Generate the parameters for a query to update a node in the database."""

        params = self.generate_node_key_params(node)
        params[
            generate_prefixed_param_name(PROPERTIES_PARAM_NAME, GENERIC_NODE_REF_NAME)
        ] = node.properties
        params[
            generate_prefixed_param_name(
                ADDITIONAL_LABELS_PARAM_NAME, GENERIC_NODE_REF_NAME
            )
        ] = node.additional_types

        return params

    def generate_node_key_params(self, node: Node, name=GENERIC_NODE_REF_NAME) -> dict:
        """Generate the parameters for a query to update a node in the database."""

        return {
            generate_prefixed_param_name(k, name): v for k, v in node.key_values.items()
        }

    @cache
    @correct_parameters
    def generate_update_relationship_operation_query_statement(
        self,
        operation: OperationOnRelationshipIdentity,
    ) -> str:
        """Generate a query to update a relationship in the database given a relationship operation."""

        match_from_node_segment = _match_node(operation.from_node, FROM_NODE_REF_NAME)
        match_to_node_segment = _match_node(operation.to_node, TO_NODE_REF_NAME)
        merge_rel_segment = _merge_relationship(operation.relationship_identity)
        return f"{match_from_node_segment} {match_to_node_segment} {merge_rel_segment}"

    def generate_update_rel_params(self, rel: Relationship) -> dict:
        """Generate the parameters for a query to update a relationship in the database."""

        params = {
            generate_prefixed_param_name(k, RELATIONSHIP_REF_NAME): v
            for k, v in rel.key_values.items()
        }
        params[
            generate_prefixed_param_name(PROPERTIES_PARAM_NAME, RELATIONSHIP_REF_NAME)
        ] = rel.properties

        return params

    def generate_update_rel_between_nodes_params(
        self, rel: RelationshipWithNodes
    ) -> dict:
        """Generate the parameters for a query to update a relationship in the database."""

        params = self.generate_update_rel_params(rel.relationship)
        params.update(self.generate_node_key_params(rel.from_node, FROM_NODE_REF_NAME))
        params.update(self.generate_node_key_params(rel.to_node, TO_NODE_REF_NAME))
        return params

    def generate_batch_update_node_operation_batch(
        self,
        operation: OperationOnNodeIdentity,
        nodes: Iterable[Node],
    ) -> QueryBatch:
        """Generate a batch of queries to update nodes in the database in the same way of the same type."""

        query_statement = self.generate_update_node_operation_query_statement(operation)
        params = [self.generate_update_node_operation_params(node) for node in nodes]
        return QueryBatch(query_statement, params)

    def generate_batch_update_relationship_query_batch(
        self,
        operation: OperationOnRelationshipIdentity,
        relationships: Iterable[RelationshipWithNodes],
    ) -> QueryBatch:
        """Generate a batch of queries to update relationships in the database in the same way of the same type."""

        query = self.generate_update_relationship_operation_query_statement(operation)
        params = [
            self.generate_update_rel_between_nodes_params(rel) for rel in relationships
        ]
        return QueryBatch(query, params)

    def generate_ttl_match_query(self, config: TimeToLiveConfiguration) -> Query:
        earliest_allowed_time = datetime.utcnow() - timedelta(
            hours=config.expiry_in_hours
        )
        params = {"earliest_allowed_time": earliest_allowed_time}
        if config.custom_query is not None:
            return Query(config.custom_query, params)

        query_builder = QueryBuilder()
        ref_name = "x"

        if config.graph_object_type == GraphObjectType.NODE:
            query_builder = query_builder.match().node(
                labels=config.object_type, ref_name=ref_name
            )
        else:
            query_builder = (
                query_builder.match()
                .node()
                .related_to(label=config.object_type, ref_name=ref_name)
                .node()
            )

        query_builder = query_builder.where_literal(
            f"{ref_name}.last_ingested_at <= $earliest_allowed_time"
        ).return_literal(f"id({ref_name}) as id")

        return Query(str(query_builder), params)

    def generate_ttl_query_from_configuration(
        self, config: TimeToLiveConfiguration
    ) -> Query:
        ttl_match_query = self.generate_ttl_match_query(config)
        operation = (
            DELETE_NODE_QUERY
            if config.graph_object_type == GraphObjectType.NODE
            else DELETE_REL_QUERY
        )
        return ttl_match_query.feed_batched_query(operation)


from django.shortcuts import redirect, render
from django.urls import reverse

from .models import Brand, Feature, Person, Rating, Project, TelegramUser
from .forms import RequestForm
from rest_framework.generics import CreateAPIView
from .serializers import TelegramBotSerializer
from telegram.ext import (
    Updater,
    CommandHandler,
    MessageHandler,
    Filters,
    ConversationHandler,
)

class TelegramUserCreateAPIView(CreateAPIView):
    serializer_class = TelegramBotSerializer
    queryset = TelegramUser.objects.all()

def home_view(request):
    brands = Brand.objects.all()
    features = Feature.objects.all()
    ratings = Rating.objects.all()
    projects = Project.objects.all()

    data = {
        'brands': brands,
        'features': features,
        'ratings': ratings,
        'projects': projects
    }

    return render(request, 'main/home.html', context=data)

def contact_us_view(request):
    return render(request, 'main/contact.html')

def success_view(request):
    return render(request, 'main/success.html')

def about_us_view(request):
    persons = Person.objects.all()
    data = {
        'persons': persons
    }
    return render(request, 'main/about-us.html', context=data)

def projects_view(request):
    projects = Project.objects.all()
    data = {
        'projects': projects
    }
    return render(request, 'main/projects.html', context=data)

def submit_request(request):
    updater = Updater("5478835430:AAEqT3Kw8QKDlTPRzIfSRVYAlBOW2uIxfPQ")
    telegram_users = TelegramUser.objects.all()
    if request.method == "POST" or None:
        form = RequestForm(request.POST or None)
        if form.is_valid():
            form.save(commit=False)
            obj = form.save()
            for user in telegram_users:
                updater.bot.send_message(chat_id=user.chat_id, text=f"""
                Ух ты у нас новая заявка № {obj.id} !!!  \n Номер телефона:  {obj.phone} \n Имя:  {obj.name}  \n Время создание: {obj.created_at} \n
                """, parse_mode="HTML")
            return redirect(reverse('success_view'))
        else:
            return render(request, 'main/contact.html', {'form': form})
    else:
        return render(request, 'main/404.html')


from  PyDSS.pyPlots.pyPlotAbstract import PlotAbstract
from bokeh.plotting import figure, curdoc
from bokeh.io import output_file
from bokeh.client import push_session
from bokeh.layouts import column
from bokeh.palettes import Set1
from ast import literal_eval as LE

class TimeSeries(PlotAbstract):
    def __init__(self,PlotProperties, dssBuses, dssObjects, dssCircuit, dssSolver):
        super(TimeSeries).__init__()
        self.__dssBuses = dssBuses
        self.__dssObjs = dssObjects
        self.__PlotProperties = PlotProperties
        self.__Properties =  LE(PlotProperties['Properties'])
        self.__index = LE(PlotProperties['index'])
        self.__multiplier = LE(PlotProperties['yScaler'])

        if self.__PlotProperties['yObjectType'] == 'Element':
            self.yObj = self.__dssObjs[self.__PlotProperties['yObjName']]
        elif self.__PlotProperties['yObjectType'] == 'Bus':
            self.yObj = self.__dssBuses[self.__PlotProperties['yObjName']]
        elif self.__PlotProperties['yObjectType'] == 'Circuit':
            self.yObj = dssCircuit

        self.Figures = []
        self.Y = [[] for x in self.__Properties]
        self.X = [[] for x in self.__Properties]
        self.Lines = [None for x in self.__Properties]
        output_file(PlotProperties['FileName'])
        for i, yProperties in enumerate(self.__Properties):
            Multiplier = self.__multiplier[i]
            pptyType ,Property = yProperties.split('.')
            if pptyType == 'p':
                newValue = self.yObj.GetParameter(Property)
            elif pptyType == 'v' and self.__PlotProperties['yObjectType'] != 'Circuit':
                newValue = self.yObj.GetVariable(Property)
            elif pptyType == 'v' and self.__PlotProperties['yObjectType'] == 'Circuit':
                newValue = getattr(self.yObj, Property)()

            if newValue is not None:
                if isinstance(newValue, list):
                    if self.__index[i] == 'SumEven':
                        self.Y[i].append(Multiplier * sum(newValue[::2]))
                    elif self.__index[i] == 'SumOdd':
                        self.Y[i].append(Multiplier * sum(newValue[1::2]))
                    elif self.__index[i] == 'Even' :
                        self.Y[i]= [[Multiplier * x] for x in newValue[::2]]
                    elif self.__index[i] == 'Odd':
                        self.Y[i] = [[Multiplier * x] for x in newValue[1::2]]
                    elif 'Index=' in self.__index[i]:
                        c = int(self.__index[i].replace('Index=', ''))
                        self.Y[i].append(Multiplier * newValue[c])
                else:
                    self.Y[i].append(Multiplier * newValue)
                self.X[i] = range(len(self.Y[i])+1)

            Figure = figure(plot_width=self.__PlotProperties['Width'],
                            plot_height=self.__PlotProperties['Height'],
                            title=self.__PlotProperties['yObjName'])
            if isinstance(self.Y[i][0], list):
                self.Lines[i] = []
                for j, Y in enumerate(self.Y[i]):
                    a = Figure.line(self.X[i], Y, line_width=3, line_alpha=1, color=Set1[9][j])
                    self.Lines[i].append(a)
            else:
                self.Lines[i] = Figure.line(self.X[i], self.Y[i], line_width=3, line_alpha=1)
            if self.__index[i] is not None:
                Figure.yaxis.axis_label = Property + '-' + self.__index[i]
            else:
                Figure.yaxis.axis_label = Property
            Figure.xaxis.axis_label = 'Timesteps'
            self.Figures.append(Figure)

        self.Layout = column(self.Figures)
        self.doc = curdoc()
        self.doc.add_root(self.Layout)
        self.doc.title = "PyDSS"
        self.session = push_session(self.doc)
        #self.session.show(self.Layout)  # open the document in a browser

        return

    def GetSessionID(self):
        return self.session.id

    def UpdatePlot(self):
        for i, yProperties in enumerate(self.__Properties):
            Multiplier = self.__multiplier[i]
            pptyType ,Property = yProperties.split('.')
            if pptyType == 'p':
                newValue = self.yObj.GetParameter(Property)
            elif pptyType == 'v' and self.__PlotProperties['yObjectType'] != 'Circuit':
                newValue = self.yObj.GetVariable(Property)
            elif pptyType == 'v' and self.__PlotProperties['yObjectType'] == 'Circuit':
                newValue = getattr(self.yObj, Property)()

            if newValue is not None:
                if isinstance(newValue, list):
                    if self.__index[i] == 'SumEven':
                        self.Y[i].append(Multiplier * sum(newValue[::2]))
                    elif self.__index[i] == 'SumOdd':
                        self.Y[i].append(Multiplier * sum(newValue[1::2]))
                    elif self.__index[i] == 'Even':
                        for j, Value in enumerate(newValue[::2]):
                            self.Y[i][j].append(Multiplier * Value)
                    elif self.__index[i] == 'Odd':
                        for j, Value in enumerate(newValue[1::2]):
                            self.Y[i][j].append(Multiplier * Value)
                    elif 'Index=' in self.__index[i]:
                        c = int(self.__index[i].replace('Index=',''))
                        self.Y[i].append(Multiplier * newValue[c])
                else:
                    self.Y[i].append(Multiplier * newValue)
                self.X[i] = range(len(self.Y[i])+1)

            if isinstance(self.Y[i][0], list):
                for j, Y in enumerate(self.Y[i]):
                    self.Lines[i][j].data_source.data['x'] = range(len(Y)+1)
                    self.Lines[i][j].data_source.data['y'] = Y
            else:
                if self.__PlotProperties['Dynamic'] and len(self.X[i]) > self.__PlotProperties['nSamples']:
                    self.X[i] = self.X[i][-self.__PlotProperties['nSamples']:]
                    self.Y[i] = self.Y[i][-self.__PlotProperties['nSamples']:]
                self.Lines[i].data_source.data['x'] = self.X[i]
                self.Lines[i].data_source.data['y'] = self.Y[i]
        return


#!/usr/bin/python
#-*- coding: utf-8 -*-

'''

'''
__author__ = 'wpxiao'

from django.shortcuts import render,HttpResponse,redirect
from web import models
from django.urls import reverse
from utils import pagination_new
def index(request,*args,**kwargs):
    article_type_list = models.Article.type_choice
    if kwargs:
        article_type_id = int(kwargs["article_type_id"])
        base_url = reverse("index",kwargs=kwargs)
        print(base_url)
    else:
        article_type_id = None
        base_url = "/"
    # 获取文章分类的数量
    data_count = article_list = models.Article.objects.filter(**kwargs).count()

    page_obj = pagination_new.Pagination(current_page=request.GET.get("p"),data_count=data_count)
    article_list = models.Article.objects.filter(**kwargs).order_by('-nid')[page_obj.start:page_obj.end]
    page_str = page_obj.page_str(base_url)
    print(request.session["user_info"])
    return render(
        request,
        'index.html',
        {
            'article_list': article_list,
            'article_type_id': article_type_id,
            'article_type_list': article_type_list,
            'page_str': page_str,
        }
    )

from web.forms import article
from django.db import transaction
from utils.xss import XSSFilter
def article_add(request):
    if request.method == "GET":
        articleForm = article.ArticleForm(request)
        return render(request,"article_add.html",{"articleForm":articleForm})
    elif request.method == "POST":
        articleForm = article.ArticleForm(request=request,data=request.POST)
        if articleForm.is_valid():
            # 开启事务操作
            with transaction.atomic():
                tags = articleForm.cleaned_data.pop("tags")
                content = articleForm.cleaned_data.pop("content")
                content = XSSFilter().process(content)
                articleForm.cleaned_data["blog_id"] = request.session["user_info"]["blog__nid"]
                #创建文章
                article_obj = models.Article.objects.create(**articleForm.cleaned_data)
                # models.ArticleDetail.objects.create(content=content,article=article_obj)
                #功能同上，创建文章详细内容
                models.ArticleDetail.objects.create(content=content, article_id=article_obj.nid)
                tag_list = []
                for tag_id in tags:
                    tag_id = int(tag_id)
                    tag_list.append(models.Article2Tag(article_id = article_obj.nid,tag_id=tag_id))
                # 批量创建文章与标签关系，如果不调用批量创建接口的话，那就需要手动save
                models.Article2Tag.objects.bulk_create(tag_list)
            return redirect("/")
        else:
            # 表单验证错误的情况
            return render(request, "article_add.html", {"articleForm": articleForm})

def article_show(request,**kwargs):
    blog_site = kwargs.pop("blog_site")
    article_nid = int(kwargs.pop("article_nid"))
    article_obj = models.Article.objects.filter(nid=article_nid,blog__site=blog_site).first()
    # 一对一关系的反向查询
    article_content = article_obj.articledetail.content
    print(article_content)
    return render(request,"article_show.html",{"article_obj":article_obj,"article_content":article_content})

def upload_img(request):
    print("..........")
    result = {"error":0,"url":None}
    if request.method == "POST":
        dir = request.GET.get("dir",None)
        if dir == "image":
            img_obj = request.FILES.get("imgFile", None)
            print("img_obj:", img_obj.name)
            import os, json
            img_path = os.path.join("static/imgs/upload_img", img_obj.name)
            with open(img_path, mode="wb") as f:
                for i in img_obj.chunks():
                    f.write(i)
                result["url"] = "/static/imgs/upload_img/" + img_obj.name
            return HttpResponse(json.dumps(result))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import sys
import time
import json
import socket
import shutil
import logging
import argparse
import subprocess
import logging.handlers
from urllib.parse import urlsplit

import requests

try:
    from config import *
except ImportError:
    from default_config import *

elogger = logging.getLogger('stderr')
ologger = logging.getLogger('stdout')

def main():
    """python3 unblockgw.py [-h]
"""
    elogger.debug("")
    elogger.debug(" ".join(sys.argv))
    Gateway.execute(sys.argv[1:])

class Gateway(object):

    @classmethod
    def execute(cls, raw_args):
        """python3 unblockgw.py {status,on,off,check,renew,setup,create}

Unblock CHN 网关命令：
  status                  查看代理状态
  on                      开启代理
  off                     关闭代理
  check <URL/IP/域名>     检查 <URL/IP/域名> 是否走代理
  renew                   更新规则
  setup [--no-ss]         一键配置网关 [--no-ss: 跳过配置 ss-redir]
  create                  仅生成 ipset 规则配置文件
"""
        parser = argparse.ArgumentParser(usage=cls.execute.__doc__)
        parser.add_argument(
            'cmd', choices=['status', 'on', 'off', 'check', 'renew', 'setup', 'create'])
        args = parser.parse_args(raw_args[0:1])

        if args.cmd == 'create':
            cls.cmd_create()
            return

        # 检查 iptables 和 ipset 命令是否存在
        cls.check_ipset_iptables()

        if args.cmd == 'on':
            cls.cmd_on()
        elif args.cmd == 'off':
            cls.cmd_off()
        elif args.cmd == 'status':
            cls.cmd_status()
        elif args.cmd == 'check':
            cls.cmd_check(raw_args[1:])
        elif args.cmd == 'renew':
            cls.cmd_renew()
        elif args.cmd == 'setup':
            cls.cmd_setup(raw_args[1:])

    @classmethod
    def cmd_status(cls):
        """查看 Unblock Youku Gateway 代理状态"""
        cls.check_setup()
        ss_redir_running = cls.check_ss_redir()
        if not ss_redir_running:
            ologger.info("ss-redir 未运行")
            return
        iptables_chn_exists = cls.check_iptables_chn()
        if iptables_chn_exists:
            ologger.info("已开启")
        else:
            ologger.info("已关闭")

    @classmethod
    def cmd_on(cls):
        """开启 Unblock CHN 代理"""
        cls.check_setup()
        ss_redir_running = cls.check_ss_redir()
        if not ss_redir_running:
            success = cls.start_ss_redir()
            if not success:
                ologger.error("✘ 无法启动 ss-redir")
                sys.exit(1)
        iptables_chn_exists = cls.check_iptables_chn()
        if not iptables_chn_exists:
            cls.add_iptables_chn()
        if ss_redir_running and iptables_chn_exists:
            ologger.info("已经开启")
        else:
            ologger.info("开启成功")

    @classmethod
    def cmd_off(cls):
        """关闭 Unblock CHN 代理"""
        cls.check_setup()
        iptables_chn_exists = cls.check_iptables_chn()
        if iptables_chn_exists:
            cls.delete_iptables_chn()
            ologger.info("关闭成功")
        else:
            ologger.info("已经关闭")

    @classmethod
    def cmd_check(cls, raw_args):
        """python3 unblockgw.py router check [-h] url
        检查 url 是否走 Unblock CHN 代理
        """
        parser = argparse.ArgumentParser(usage=cls.cmd_check.__doc__)
        parser.add_argument('url', help="URL / IP / 域名")
        args = parser.parse_args(raw_args)

        cls.check_setup()

        if "://" in args.url:
            domain = urlsplit(args.url).hostname
        else:
            domain = args.url.split('/')[0]
        ip = socket.gethostbyname(domain)

        cmd = "ipset test chn {}".format(ip)
        returncode = subprocess.call(
            cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        if returncode == 0:
            ologger.info("{} 走代理".format(ip))
        else:
            ologger.info("{} 不走代理".format(ip))

    @classmethod
    def cmd_renew(cls):
        """更新 Unblock CHN 规则"""
        cls.check_setup()

        # 生成网关配置文件
        unblock_youku = UnblockYouku()
        cls.create_conf_files(unblock_youku.black_domains)

        # 清空 ipset 的 chn 表
        cmd = "ipset flush chn"
        subprocess.check_call(cmd, shell=True)
        elogger.info("✔ 清空 ipset 的 chn 表：{}".format(cmd))

        # 载入 ipset 规则
        headless_ipset_conf_path = os.path.join(DIR_PATH, "configs/ipset.headless.rules")
        cmd = "ipset restore < {}".format(headless_ipset_conf_path)
        subprocess.check_call(cmd, shell=True)
        elogger.info("✔ 载入 ipset 规则：{}".format(cmd))

        ologger.info("更新成功")

    @classmethod
    def cmd_setup(cls, raw_args):
        """python3 unblockgw.py router setup [-h] [--no-ss]
        Unblock CHN 一键配置网关
        """
        parser = argparse.ArgumentParser(usage=cls.cmd_setup.__doc__)
        parser.add_argument('--no-ss', action='store_true', help="跳过配置 ss-redir")
        args = parser.parse_args(raw_args)

        # 不跳过配置 ss-redir
        if not args.no_ss:
            # 配置 ss-redir
            cls.setup_ss_redir()

        # 生成网关配置文件
        unblock_youku = UnblockYouku()
        cls.create_conf_files(unblock_youku.black_domains)

        # 清空 ipset 的 chn 表
        cmd = "ipset flush"
        subprocess.check_call(cmd, shell=True)
        elogger.info("✔ 清空 ipset：{}".format(cmd))

        # 配置 ipset 和 iptables
        cls.setup_ipset_iptables()

        # 添加每日更新规则的 cron 定时任务
        cls.add_renew_cron_job()

        ologger.info("配置成功")

    @classmethod
    def cmd_create(cls):
        """仅生成 ipset 规则配置文件"""

        # 生成网关配置文件
        unblock_youku = UnblockYouku()
        cls.create_conf_files(unblock_youku.black_domains)

        ologger.info("生成配置文件成功")

    @classmethod
    def create_conf_files(cls, domains):
        """生成网关 ipset 规则配置文件"""

        # 生成 ipset 规则
        ipset_rules = []
        dnsmasq_rules = []
        if domains:
            for domain in domains:
                if re.match(r"\d+\.\d+\.\d+\.\d+", domain):  # IP
                    rule = "add chn {}".format(domain)
                    ipset_rules.append(rule)
                else:  # 域名
                    try:
                        ips = socket.gethostbyname_ex(domain)[2]
                        for ip in ips:
                            if ip not in ["127.0.0.1", "0.0.0.1"]:
                                rule = f"add chn {ip}"
                                ipset_rules.append(rule)
                    except:
                        pass
        ipset_rules = set(ipset_rules) #remove the same IPs
        # 从模板生成 ipset 规则配置文件 ipset.rules
        cls.create_ipset_conf_file(ipset_rules)

    @classmethod
    def create_ipset_conf_file(cls, ipset_rules):
        """从模板生成 ipset 规则配置文件 ipset.rules"""
        ipset_tpl_path = os.path.join(DIR_PATH, "configs/ipset.rules.tpl")
        if os.path.isfile(ipset_tpl_path):
            with open(ipset_tpl_path, 'r', encoding='utf-8') as f:
                ipset_tpl = f.read()
        else:
            ipset_tpl = "{rules}"
            with open(ipset_tpl_path, 'w', encoding='utf-8') as f:
                f.write(ipset_tpl)
            elogger.info("✔ 生成 ipset 默认配置模板文件：ipset.rules.tpl")

        # 无 ipset 规则 & 无自定义模板内容
        if (not ipset_rules) and (ipset_tpl == "{rules}"):
            return False

        if ipset_rules:
            ipset_rules = "\n".join(ipset_rules)
        else:
            ipset_rules = ""
        ipset_conf = ipset_tpl.format(rules=ipset_rules)

        # 生成包含表创建命令的 ipset 规则配置文件 ipset.rules
        ipset_conf_path = os.path.join(DIR_PATH, "configs/ipset.rules")
        with open(ipset_conf_path, 'w', encoding='utf-8') as f:
            f.write("create chn hash:ip family inet hashsize 1024 maxelem 65536\n")
            f.write(ipset_conf)

        # 生成不包含表创建命令的 ipset 规则配置文件 ipset.headless.rules
        headless_ipset_conf_path = os.path.join(DIR_PATH, "configs/ipset.headless.rules")
        with open(headless_ipset_conf_path, 'w', encoding='utf-8') as f:
            f.write(ipset_conf)

        elogger.info("✔ 生成 ipset 配置文件：ipset.rules & ipset.headless.rules")
        return True

    @classmethod
    def setup_ss_redir(cls):
        """配置 ss-redir"""

        # 生成 ss-redir 配置文件
        conf = SS_REDIR_CONF
        if conf['server'] is None:
            conf['server'] = input("Shadowsocks 服务器地址：").strip()
        if conf['server_port'] is None:
            conf['server_port'] = int(input("Shadowsocks 服务器端口：").strip())
        if conf['password'] is None:
            conf['password'] = input("Shadowsocks 密码：").strip()
        if conf['method'] is None:
            conf['method'] = input("Shadowsocks 加密方法：").strip()
        with open(SS_REDIR_CONF_PATH, 'w', encoding='utf-8') as f:
            json.dump(conf, f, indent=4)
        elogger.info("✔ 保存 ss-redir 配置文件：{}".format(SS_REDIR_CONF_PATH))

        # 启动 ss-redir
        success = cls.start_ss_redir()
        if not success:
            cmd = "{} -c {}".format(SS_REDIR_PATH, SS_REDIR_CONF_PATH)
            elogger.error("✘ 无法启动 ss-redir，请手动运行以下命令查看错误信息：\n{}".format(cmd))
            sys.exit(1)

        # 保存 ss-redir 启动命令到网关的 services-start 启动脚本中
        cmd = "{} -c {} -f {}"
        cmd = cmd.format(SS_REDIR_PATH, SS_REDIR_CONF_PATH, SS_REDIR_PID_PATH)
        comment = "# ss-redir"
        cls.append_to_script(SERVICES_START_SCRIPT_PATH, comment, cmd)
        elogger.info("✔ 保存 ss-redir 启动命令到网关的 services-start 启动脚本中：{}".format(SERVICES_START_SCRIPT_PATH))

    @classmethod
    def setup_ipset_iptables(cls):
        """配置 ipset 和 iptables"""
        # 载入 ipset 规则
        ipset_cmd = "ipset restore < {}".format(IPSET_CONF_PATH)
        try:
            subprocess.check_output(ipset_cmd, shell=True, stderr=subprocess.PIPE)
        except subprocess.CalledProcessError as e:
            if "set with the same name already exists" not in str(e.stderr):
                raise e
        elogger.info("✔ 载入 ipset 规则：{}".format(ipset_cmd))

        # 保存 ipset 载入命令到网关的 nat-start 启动脚本中
        comment = "# Load ipset rules"
        cls.append_to_script(NAT_START_SCRIPT_PATH, comment, ipset_cmd)
        elogger.info("✔ 保存 ipset 载入命令到网关的 nat-start 启动脚本中：{}".format(NAT_START_SCRIPT_PATH))

        # 添加 iptables 规则
        iptables_chn_exists = cls.check_iptables_chn()
        if not iptables_chn_exists:
            cls.add_iptables_chn()
        elogger.info("✔ 添加 iptables 规则：{}".format(ADD_IPTABLES_CHN_CMD))

        # 保存 iptables 添加规则命令到网关的 nat-start 启动脚本中
        comment = "# Redirect chn ipset to ss-redir"
        cmd = 'if [ "$(nvram get unblockgw_on)" = "True" ]; then {}; fi'
        cmd = cmd.format(ADD_IPTABLES_CHN_CMD)
        cls.append_to_script(NAT_START_SCRIPT_PATH, comment, cmd)
        elogger.info("✔ 保存 iptables 规则添加命令到启动脚本中：{}".format(NAT_START_SCRIPT_PATH))

    @classmethod
    def add_renew_cron_job(cls):
        """添加每日更新规则的 cron 定时任务"""
        renew_cmd = "0 {} * * * {} {} renew\r\n"
        unblockgw_path = os.path.realpath(__file__)
        renew_cmd = renew_cmd.format(RENEW_TIME, PYTHON3_PATH, unblockgw_path)

        # 写入定时任务到文件中
        ipset_tpl_path = os.path.join(DIR_PATH, "configs/renew_task")
        with open(ipset_tpl_path, 'w', encoding='utf-8') as f:
            f.write(renew_cmd)
        elogger.info("✔ 生成 corn 任务配置模板文件：renew_task")

        cron_cmd = "crontab configs/renew_task"
        try:
            subprocess.check_call(cron_cmd, shell=True)
        except subprocess.CalledProcessError as e:
            elogger.exception(e)
            elogger.warning("✘ 无法添加每日更新规则的定时任务，你需要手动添加以下条目到 crontab 中：\n{}".format(renew_cmd))
            return
        else:
            elogger.info("✔ 定时每日 {} 点更新规则：{}".format(RENEW_TIME, cron_cmd))

        # 保存以上定时命令到网关的 services-start 启动脚本中
        comment = "# unblockgw_renew cron job"
        cls.append_to_script(SERVICES_START_SCRIPT_PATH, comment, cron_cmd)
        elogger.info("✔ 保存定时更新规则命令到网关的 services-start 启动脚本中：{}".format(SERVICES_START_SCRIPT_PATH))

    @classmethod
    def remove_renew_cron_job(cls):
        """移除每日更新规则的 cron 定时任务"""
        cmd = "cru d unblockgw_renew"
        subprocess.check_call(cmd, shell=True)
        elogger.info("✔ 删除每日更新规则的 cron 定时任务：{}".format(cmd))

        # 从启动脚本里移除定时命令
        comment = "# unblockgw_renew cron job"
        cls.remove_from_script(SERVICES_START_SCRIPT_PATH, comment)
        elogger.info("✔ 从启动脚本里移除定时命令：{}".format(SERVICES_START_SCRIPT_PATH))

    @classmethod
    def start_ss_redir(cls):
        """启动 ss-redir"""
        cmd = "{} -c {} -f {}"
        cmd = cmd.format(SS_REDIR_PATH, SS_REDIR_CONF_PATH, SS_REDIR_PID_PATH)
        subprocess.call(cmd, shell=True)
        time.sleep(1)
        is_running = cls.check_ss_redir()
        if is_running:
            elogger.info("✔ 启动 ss-redir：{}".format(cmd))
        return is_running

    @classmethod
    def stop_ss_redir(cls):
        """停止 ss-redir"""
        with open(SS_REDIR_PID_PATH, 'r', encoding='utf-8') as f:
            pid = f.read()
        cmd = "kill {}".format(pid)
        subprocess.check_call(cmd, shell=True)
        elogger.info("✔ 停止 ss-redir：{}".format(cmd))

    @classmethod
    def check_ss_redir(cls):
        """检查 ss-redir 是否运行中"""
        with open(SS_REDIR_PID_PATH, 'r', encoding='utf-8') as f:
            pid = f.read()
        return os.path.exists("/proc/{}".format(pid))

    @classmethod
    def check_setup(cls):
        """检查网关是否配置过"""
        chn_ipset_exists = cls.check_chn_ipset()
        if not chn_ipset_exists:
            ologger.error("✘ 网关未正确配置，请先运行以下命令进行配置：\npython3 unblockgw.py setup")
            sys.exit(1)

    @classmethod
    def check_ipset_iptables(cls):
        """检查 iptables 和 ipset 命令是否存在"""
        iptables_exists = cls.check_command('iptables')
        ipset_exists = cls.check_command('ipset')
        if not (iptables_exists and ipset_exists):
            d = {'iptables': iptables_exists, 'ipset': ipset_exists}
            missing = [k for k in d if not d[k]]
            ologger.error("✘ 运行环境不支持 {} 命令".format(" 和 ".join(missing)))
            sys.exit(1)

    @classmethod
    def check_chn_ipset(cls):
        """检查 ipset 是否有 chn 表"""
        cmd = "ipset list chn"
        returncode = subprocess.call(
            cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        return returncode == 0

    @classmethod
    def add_iptables_chn(cls):
        """iptables 添加 chn ipset 规则"""
        subprocess.check_call(ADD_IPTABLES_CHN_CMD, shell=True)

    @classmethod
    def delete_iptables_chn(cls):
        """iptables 删除 chn ipset 规则"""
        subprocess.check_call(DELETE_IPTABLES_CHN_CMD, shell=True)

    @classmethod
    def check_iptables_chn(cls):
        """检查 iptables 是否有 chn ipset 规则"""
        returncode = subprocess.call(
            CHECK_IPTABLES_CHN_CMD, shell=True, stderr=subprocess.DEVNULL)
        return returncode == 0

    @classmethod
    def append_to_script(cls, script_path, comment, cmd):
        """添加命令到脚本"""
        if os.path.isfile(script_path):
            with open(script_path, 'r', encoding='utf-8') as f:
                scpt = f.read()
        else:
            scpt = "#!/bin/sh\n"
        if comment not in scpt:
            scpt += "\n" + comment + "\n" + cmd + "\n"
        with open(script_path, 'w', encoding='utf-8') as f:
            f.write(scpt)
        cmd = "chmod a+rx {}".format(script_path)
        subprocess.check_call(cmd, shell=True)

    @classmethod
    def remove_from_script(cls, script_path, comment):
        """从脚本中移除命令"""
        if not os.path.isfile(script_path):
            return
        with open(script_path, 'r', encoding='utf-8') as f:
            scpt = f.read()
        pattern = r"\n" + comment + r"\n.+\n?"
        scpt = re.sub(pattern, "", scpt)
        with open(script_path, 'w', encoding='utf-8') as f:
            f.write(scpt)

    @classmethod
    def check_command(cls, command):
        """检查命令是否存在"""
        returncode = subprocess.call(
            ["which", command],
            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        if returncode == 0:
            return True
        else:
            return False

class UnblockYouku(object):

    def __init__(self):
        super(UnblockYouku, self).__init__()
        self.source = requests.get(UNBLOCK_YOUKU_URLSJS_URL).text
        self._black_urls = None
        self._white_urls = None
        self._black_domains = None
        self._white_domains = None

    @property
    def black_urls(self):
        """URLs 黑名单"""
        if self._black_urls is not None:
            return self._black_urls

        header_urls = self.extract('header_urls')
        redirect_urls = self.extract('redirect_urls')
        chrome_proxy_urls = self.extract('chrome_proxy_urls')
        pac_proxy_urls = self.extract('pac_proxy_urls')

        self._black_urls = header_urls + redirect_urls + chrome_proxy_urls + pac_proxy_urls
        self._black_urls = list(set(self._black_urls))
        self._black_urls.sort()

        return self._black_urls

    @property
    def white_urls(self):
        """URLs 白名单"""
        if self._white_urls is not None:
            return self._white_urls

        chrome_proxy_bypass_urls = self.extract('chrome_proxy_bypass_urls')
        pac_proxy_bypass_urls = self.extract('pac_proxy_bypass_urls')

        self._white_urls = chrome_proxy_bypass_urls + pac_proxy_bypass_urls
        self._white_urls = list(set(self._white_urls))
        self._white_urls.sort()

        return self._white_urls

    @property
    def black_domains(self):
        """域名黑名单"""
        if self._black_domains is not None:
            return self._black_domains

        self._black_domains = []
        for url in self.black_urls:
            domain = urlsplit(url).hostname
            self._black_domains.append(domain)

        self._black_domains = list(set(self._black_domains))
        self._black_domains.sort(key=lambda s: s[::-1], reverse=True)

        return self._black_domains

    @property
    def white_domains(self):
        """域名白名单"""
        if self._white_domains is not None:
            return self._white_domains

        self._white_domains = []
        for url in self.white_urls:
            domain = urlsplit(url).hostname
            self._white_domains.append(domain)

        self._white_domains = list(set(self._white_domains))
        self._white_domains.sort(key=lambda s: s[::-1], reverse=True)

        return self._white_domains

    def extract(self, name):
        """从 Unblock Youku 的 urls.js 中提取指定的 URL 列表"""
        pattern = "unblock_youku\\.{}\\s*=.+?(\\[.+?\\])".format(name)
        match = re.search(pattern, self.source, re.DOTALL)
        if not match:
            elogger.error("✘ 从 Unblock Youku 提取 {} 规则失败".format(name))
            sys.exit(1)
        s = match.group(1)
        s = s.replace("'", '"')  # 替换单引号为双引号
        s = re.sub(r"(?<!:)//.+", "", s)  # 去除注释
        s = re.sub(r",\s*\]", "\n]", s)  # 去除跟在最后一个元素后面的逗号
        urls = json.loads(s)
        return urls

def init_logging():
    """日志初始化"""

    root_logger = logging.getLogger()
    root_logger.setLevel(logging.NOTSET)

    # 输出日志到文件
    log_file_path = os.path.join(DIR_PATH, "unblockgw.log")
    formatter = logging.Formatter(
        "%(asctime)s ~ %(levelname)-8s - "
        "%(message)s",
        datefmt="%Y-%m-%d %H:%M:%S"
    )
    logfile = logging.handlers.RotatingFileHandler(
        log_file_path,
        maxBytes=10 * 1024 * 1024,
        backupCount=1
    )
    logfile.setFormatter(formatter)
    logfile.setLevel(logging.DEBUG)
    root_logger.addHandler(logfile)

    # 输出日志到控制台
    formatter = logging.Formatter("%(message)s")
    # stderr logger
    console = logging.StreamHandler(sys.stderr)
    console.setFormatter(formatter)
    console.setLevel(logging.INFO)
    elogger.addHandler(console)
    # stdout logger
    console = logging.StreamHandler(sys.stdout)
    console.setFormatter(formatter)
    console.setLevel(logging.INFO)
    ologger.addHandler(console)

    # 设置 requests 和 urllib3 日志级别
    requests_logger = logging.getLogger("requests")
    requests_logger.setLevel(logging.WARNING)
    urllib3_logger = logging.getLogger('urllib3')
    urllib3_logger.setLevel(logging.WARNING)

if __name__ == '__main__':
    try:
        init_logging()
        main()
    except Exception as error:
        elogger.exception(error)


import subprocess

import weave
from weave import monitoring

import publish_dataset

import board_datasets
import board_models
import board_model_compare

import model_basic
import model_hf

from weaveflow import settings

def main():
    dataset_ref = weave.storage.publish(
        publish_dataset.read_dataset(), f"{settings.project}/dataset"
    )
    print("published dataset")

    weave.publish(
        board_datasets.make_board(settings.entity, settings.project),
        f"{settings.project}/datasets",
    )
    print("published datasets board")

    model_a_ref = weave.storage.publish(
        model_basic.BasicModel({}),
        f"{settings.project}/PredictBasic",
    )

    # Make some predictions to ensure we have some in the prediction
    # StreamTable
    monitoring.init_monitor(
        f"{settings.entity}/{settings.project}/{settings.predictions_stream_name}"
    )
    model_a = weave.storage.get(model_a_ref)
    weave.use(model_a.predict("hello"))
    weave.use(model_a.predict("my name is jim"))

    print("published model a")
    subprocess.run(
        [
            "python",
            "op_evaluate.py",
            "--dataset",
            str(dataset_ref),
            "--model",
            str(model_a_ref),
        ],
        check=True,
    )
    print("evaluated model a")

    model_b_ref = weave.storage.publish(
        model_hf.HFModel({"model_name": "bert-base-uncased"}),
        f"{settings.project}/PredictBasic",
    )

    # Make predictions
    model_b = weave.storage.get(model_a_ref)
    weave.use(model_b.predict("my name is bob"))

    print("published model b")
    subprocess.run(
        [
            "python",
            "op_evaluate.py",
            "--dataset",
            str(dataset_ref),
            "--model",
            str(model_b_ref),
        ],
        check=True,
    )
    print("evaluated model b")

    # model_c = weave.publish(
    #     model_basic({"shares_skip_chars": 5, "name_up_to_period": True}), "ModelBasic"
    # )
    # cli.run_op(op_evaluate.evaluate_multi_task_f1(dataset, model_c))

    weave.publish(
        board_models.make_board(
            settings.entity, settings.project, settings.predictions_stream_name
        ),
        f"{settings.project}/models",
    )
    print("published models board")

    weave.publish(
        board_model_compare.make_board(settings.entity, settings.project),
        f"{settings.project}/model_compare",
    )
    print("published model compare board")

if __name__ == "__main__":
    main()


import gi.repository
gi.require_version('Budgie', '1.0')
from gi.repository import Budgie, GObject, Gtk, GdkPixbuf, GLib, Gio, Gdk
import time
import os
import svgwrite
import datetime
from createsvg import createSVG
from math import sin, cos, pi

"""
    Analog Clock Applet for the Budgie Panel

    Copyright © 2020 Samuel Lane
    http://github.com/samlane-ma/

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""

""" Some "constants":
    Clock is drawn at 200x200 px resolution, with a center at 100,100 and
    then scaled to fit panel, but these can be played around with to fine
    tune the clocks appearance.
"""
MAXIMUM_SIZE       = 200  # 200 default
MINIMUM_SIZE       =  22  #  22 default

IMAGE_SIZE         = 200  # 200 default
X_CENTER           = 100  #  50 default
Y_CENTER           = 100  #  50 default
CLOCK_RADIUS       =  92  #  46 default
HOUR_HAND_LENGTH   =  56  #  28 default
MINUTE_HAND_LENGTH =  76  #  38 default
FRAME_THICKNESS    =  10  #   5 default
HAND_THICKNESS     =  10  #   5 default
CENTER_DOT         =   8  #   3 default
MARKING_THICKNESS  =   8  #   4 default
MARKING_LENGTH     =  16  #   8 default

UPDATE_INTERVAL    =   5  #   5 default (in seconds)
FORCE_REDRAW       =  -1  # This is just to clarify why -1 is used in the code

app_settings = Gio.Settings.new("com.github.samlane-ma.budgie-analog-clock")
path = "com.solus-project.budgie-panel"

class BudgieAnalogClock(GObject.GObject, Budgie.Plugin):
    """ This is simply an entry point into your Budgie Applet implementation.
        Note you must always override Object, and implement Plugin.
    """
    # Good manners, make sure we have unique name in GObject type system
    __gtype_name__ = "BudgieAnalogClock"

    def __init__(self):
        """ Initialisation is important.
        """
        GObject.Object.__init__(self)

    def do_get_panel_widget(self, uuid):
        """ This is where the real fun happens. Return a new Budgie.Applet
            instance with the given UUID. The UUID is determined by the
            BudgiePanelManager, and is used for lifetime tracking.
        """
        return BudgieAnalogClockApplet(uuid)

class BudgieAnalogClockSettings(Gtk.Grid):

    def __init__(self, setting):
        super().__init__()

        self.label_text = ["", "Clock Size (px)","Frame Color","Hands Color",
                           "Face Color","Transparent face","","Show hour marks"]
        self.setting_name = ["clock-outline","clock-hands","clock-face"]

        for n in range(8):
            label = Gtk.Label()
            label.set_text(self.label_text[n])
            label.set_halign(Gtk.Align.START)
            label.set_valign(Gtk.Align.CENTER)
            self.attach(label, 0, n, 1, 1)

        spin_clock_size_adj = Gtk.Adjustment(value=app_settings.get_int("clock-size"),
                                  lower=MINIMUM_SIZE, upper=MAXIMUM_SIZE, step_incr=1)
        spin_clock_size = Gtk.SpinButton()
        spin_clock_size.set_adjustment(spin_clock_size_adj)
        spin_clock_size.set_digits(0)
        self.attach(spin_clock_size, 1, 1, 1, 1)

        self.colorbuttons = []
        for n in range(3):
            load_color = app_settings.get_string(self.setting_name[n])
            color = Gdk.RGBA()
            if load_color == "none":
                color.parse("rgba(0,0,0,0)")
            else:
                color.parse(load_color)
            button = Gtk.ColorButton.new_with_rgba(color)
            button.connect("color_set",self.on_color_changed,self.setting_name[n])
            self.colorbuttons.append(button)
            self.attach(self.colorbuttons[n], 1, n+2, 1, 1)

        button_set_transparent = Gtk.Button.new_with_label("Set")
        button_set_transparent.connect("clicked",self.on_set_transparent)
        self.attach(button_set_transparent, 1, 5, 1, 1)
        switch_markings = Gtk.Switch()
        switch_markings.set_halign(Gtk.Align.END)
        self.attach(switch_markings, 1, 7, 1, 1)

        app_settings.bind("clock-size",spin_clock_size,"value",Gio.SettingsBindFlags.DEFAULT)
        app_settings.bind("draw-marks",switch_markings,"active",Gio.SettingsBindFlags.DEFAULT)

        self.show_all()

    def on_set_transparent(self, button):
        self.colorbuttons[2].set_alpha(0)
        app_settings.set_string("clock-face","none")

    def on_color_changed(self, button, clock_part):
        color = button.get_color()
        hex_code = "#{:02x}{:02x}{:02x}".format(int(color.red/256),
                                                int(color.green/256),
                                                int(color.blue/256))
        app_settings.set_string(clock_part, hex_code)

class BudgieAnalogClockApplet(Budgie.Applet):
    """ Budgie.Applet is in fact a Gtk.Bin """
    manager = None

    def __init__(self, uuid):

        Budgie.Applet.__init__(self)

        self.uuid = uuid
        self.keep_running = True

        user = os.environ["USER"]
        self.tmp = os.path.join("/tmp/", user + "_panel_analog_clock.svg")
        self.max_size = MAXIMUM_SIZE

        self.load_settings()

        self.box = Gtk.Box()
        self.add(self.box)
        self.clock_image = Gtk.Image()
        self.box.add(self.clock_image)
        self.show_all()
        self.app_signal_id = app_settings.connect("changed",self.on_settings_change)
        GLib.timeout_add_seconds(UPDATE_INTERVAL, self.update_time)

        # This triggers the applet to shut down when removed from the panel
        # It needs a slight delay to work correctly
        GObject.idle_add(self.watch_applet, self.uuid)

    def find_applet (self, check_uuid, applets):
        for find_uuid in applets:
            if find_uuid == check_uuid:
                return True
        return False

    def watch_applet (self, check_uuid):
        applets = []
        panel_settings = Gio.Settings(path);
        allpanels_list = panel_settings.get_strv("panels")
        for p in allpanels_list:
            panelpath = "/com/solus-project/budgie-panel/panels/{" + p + "}/"
            self.currpanelsubject_settings = Gio.Settings.new_with_path(path + ".panel", panelpath)
            applets = self.currpanelsubject_settings.get_strv("applets")
            if self.find_applet(check_uuid, applets):
                # Need this signal id to disconnect it on quit
                self.panel_signal_id = self.currpanelsubject_settings.connect(
                              "changed::applets", self.is_applet_running, check_uuid)
        return False

    def is_applet_running (self, arg1, arg2, check_uuid):
        applets =self.currpanelsubject_settings.get_strv("applets")
        if not self.find_applet(check_uuid, applets):
            # Disconnect the signals
            self.currpanelsubject_settings.disconnect(self.panel_signal_id)
            app_settings.disconnect(self.app_signal_id)
            self.keep_running = False

    def do_panel_position_changed(self,position):
        if position == Budgie.PanelPosition.TOP or position == Budgie.PanelPosition.BOTTOM:
            self.box.set_orientation(Gtk.Orientation.HORIZONTAL)
        else:
            self.box.set_orientation(Gtk.Orientation.VERTICAL)
        self.update_clock()

    def do_panel_size_changed(self,panel_size,icon_size,small_icon_size):
        # Keeps the clock smaller than panel, but no smaller than MINIMUM_SIZE
        self.max_size = panel_size - 6
        if self.max_size < MINIMUM_SIZE:
            self.max_size = MINIMUM_SIZE
        current_size = app_settings.get_int("clock-size")
        # Don't let the clock get bigger than the Budgie Panel
        if current_size > self.max_size:
            self.clock_scale = self.max_size
        self.update_clock()

    def validate_settings(self):
        """ Reset invalid colors to defaults - "none" is a valid color name.
            Invalid settings (hopefully) should really only ever occur if
            the dconf values are manually set to something wrong
        """
        setting_name = ["clock-hands", "clock-outline", "clock-face"]
        default_color = ["#000000", "#000000", "#FFFFFF"]
        for n in range(3):
            testcolor = Gdk.RGBA()
            colorname = app_settings.get_string(setting_name[n])
            if (colorname != "none") and (not testcolor.parse(colorname)):
                app_settings.set_string(setting_name[n],default_color[n])

    def on_settings_change(self, *args):
        self.load_settings()
        self.update_clock()

    def load_settings(self):
        self.clock_scale = app_settings.get_int("clock-size")
        if self.clock_scale > self.max_size:
            self.clock_scale = self.max_size
        self.hands_color = app_settings.get_string("clock-hands")
        self.line_color = app_settings.get_string("clock-outline")
        self.fill_color = app_settings.get_string("clock-face")
        self.draw_hour_marks = app_settings.get_boolean("draw-marks")

    def update_clock(self):
        self.old_minute = FORCE_REDRAW
        self.validate_settings()
        self.update_time()

    def update_time(self):
        self.current_time = datetime.datetime.now()
        # Don't redraw unless time (minute) has changed
        if self.current_time.minute != self.old_minute:
            """ In the rare instance where the time zone is changed while applet
                is running, tzset will ensure the clock recognizes the change.
                Possibly may also be needed to recognize daylight savings change?
            """
            time.tzset()
            self.old_minute = self.current_time.minute
            self.create_clock_image(self.current_time.hour, self.current_time.minute)
            GObject.idle_add(self.load_new_image)
        return self.keep_running

    def load_new_image(self):
        self.clock_image.set_tooltip_text(self.current_time.strftime("%a %x"))
        self.pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(self.tmp, self.clock_scale,
                                                              self.clock_scale, True)
        self.clock_image.set_from_pixbuf(self.pixbuf)

    def create_clock_image (self, hours, mins):
        # If time is PM
        if hours > 12:
            hours -= 12
        # Treat hour hand like minute hand so it can be between hour markings
        hours = hours * 5 + (mins / 12)
        clock_svg = createSVG(self.tmp, IMAGE_SIZE, IMAGE_SIZE)
        # Draw an outside circle for the clock, and a small circle at the base of the hands
        clock_svg.add_circle(X_CENTER, Y_CENTER, CLOCK_RADIUS, fill=self.fill_color,
                            stroke=self.line_color, stroke_width=FRAME_THICKNESS)
        clock_svg.add_circle(X_CENTER, Y_CENTER, CENTER_DOT, stroke=self.hands_color,
                            fill=self.hands_color, stroke_width=1)

        # We are going to add hour markings around the outside edge of the clock
        if self.draw_hour_marks:
            for markings in range(12):
                if markings % 3 != 0:
                    mark_size = MARKING_THICKNESS * 2 / 3
                    mark_length = MARKING_LENGTH * 2 / 3
                else:
                    mark_size = MARKING_THICKNESS
                    mark_length = MARKING_LENGTH
                mark_x_start, mark_y_start = self.get_clock_hand_xy(markings * 5, CLOCK_RADIUS)
                mark_x_end, mark_y_end = self.get_clock_hand_xy(markings * 5, CLOCK_RADIUS - mark_length)
                clock_svg.add_line(mark_x_start, mark_y_start, mark_x_end, mark_y_end,
                                  stroke=self.line_color, stroke_width=mark_size)
        # Draw the minute and hour hands from the center to the calculated points
        hour_hand_x, hour_hand_y = self.get_clock_hand_xy (hours, HOUR_HAND_LENGTH)
        minute_hand_x, minute_hand_y = self.get_clock_hand_xy (mins, MINUTE_HAND_LENGTH)
        clock_svg.add_line(X_CENTER, Y_CENTER, hour_hand_x, hour_hand_y,
                          stroke=self.hands_color, stroke_width=HAND_THICKNESS)
        clock_svg.add_line(X_CENTER, Y_CENTER, minute_hand_x, minute_hand_y,
                          stroke=self.hands_color, stroke_width=HAND_THICKNESS)
        clock_svg.write_svg()

    def get_clock_hand_xy (self, hand_position,length):
        """ This fixes the issue that 0 degrees on a cirlce is actually 3:00
            on a clock, not 12:00 -essentially rotates the hands 90 degrees
        """
        hand_position -= 15
        if hand_position < 0:
            hand_position += 60
        # And here is how we determine the x and y coordinate to draw to
        radians = (hand_position * (pi * 2) / 60)
        x_position = round (X_CENTER + length * cos(radians))
        y_position = round (Y_CENTER + length * sin(radians))
        return x_position, y_position

    def do_supports_settings(self):
        """Return True if support setting through Budgie Setting,
        False otherwise.
        """
        return True

    def do_get_settings_ui(self):
        """Return the applet settings with given uuid"""
        return BudgieAnalogClockSettings(self.get_applet_settings(self.uuid))


from typing import Any, Mapping, List, Union
from logging import Logger
from core.messagebus import Publisher
from core.rpc import Caller, Server
from core.repo import SearchFilter
from module.auth.component.model.user_model import user_model
from module.auth.schema.user import UserData, UserLogin
from module.auth.schema.token import AccessTokenData

def register_rpc(
    logger: Logger,
    rpc_server: Server,
    rpc_caller: Caller,
    publisher: Publisher
):
    logger.info('ðŸ¥ª Register RPC handlers for "auth.user"')

    @rpc_server.register('auth_user_is_admin')
    async def user_is_admin(id: str) -> bool:
        '''
        Used by RPC Authenticator
        '''
        return await user_model.is_admin(id)

    @rpc_server.register('auth_user_is_guest')
    async def user_is_guest(id: str) -> bool:
        '''
        Used by RPC Authenticator
        '''
        return await user_model.is_guest(id)

    @rpc_server.register('auth_is_user_authorized')
    async def is_user_having_permission(
        id: str, permission_name: Union[str, List[str]]
    ) -> Mapping[str, bool]:
        '''
        Used by RPC Authenticator
        '''
        if isinstance(permission_name, str):
            return await user_model.is_authorized(id, permission_name)
        return await user_model.is_authorized(id, *permission_name)

    @rpc_server.register('auth_create_token')
    async def create_token(login_data: Mapping[str, str]) -> Mapping[str, str]:
        result = await user_model.create_auth_token(UserLogin(**login_data))
        return result.dict()

    @rpc_server.register('auth_refresh_token')
    async def refresh_token(
        refresh_token: str, access_token: str
    ) -> Mapping[str, str]:
        result = await user_model.refresh_auth_token(
            refresh_token, access_token
        )
        return result.dict()

    @rpc_server.register('auth_get_user')
    async def get(
        keyword: str,
        criterion: Mapping[str, Any],
        limit: int,
        offset: int,
        user_token_data: Mapping[str, Any]
    ) -> Mapping[str, Any]:
        result = await user_model.get(
            search_filter=SearchFilter(
                keyword=keyword, criterion=criterion
            ),
            limit=limit,
            offset=offset
        )
        return result.dict()

    @rpc_server.register('auth_get_user_by_id')
    async def get_by_id(
        id: str,
        user_token_data: Mapping[str, Any] = {}
    ) -> Mapping[str, Any]:
        row = await user_model.get_by_id(id)
        return row.dict()

    @rpc_server.register('auth_insert_user')
    async def insert(
        data: Mapping[str, Any],
        user_token_data: Mapping[str, Any]
    ) -> Mapping[str, Any]:
        user_token_data = AccessTokenData(**user_token_data)
        data['created_by'] = user_token_data.user_id
        data['updated_by'] = user_token_data.user_id
        row = await user_model.insert(
            data=UserData(**data)
        )
        return row.dict()

    @rpc_server.register('auth_update_user')
    async def update(
        id: str,
        data: Mapping[str, Any],
        user_token_data: Mapping[str, Any]
    ) -> Mapping[str, Any]:
        user_token_data = AccessTokenData(**user_token_data)
        data['updated_by'] = user_token_data.user_id
        row = await user_model.update(
            id=id, data=UserData(**data)
        )
        return row.dict()

    @rpc_server.register('auth_delete_user')
    async def delete(
        id: str,
        user_token_data: Mapping[str, Any]
    ) -> Mapping[str, Any]:
        user_token_data = AccessTokenData(**user_token_data)
        row = await user_model.delete(id=id)
        return row.dict()


import tweepy
from config.config import Config
from tweepy import OAuthHandler
import pandas as pd
import datetime as dt
import traceback

consumer_key = Config['twitter']['consumer_key']
consumer_secret=Config['twitter']['consumer_secret']
access_tocken=Config['twitter']['access_tocken']
access_secret = Config['twitter']['access_secret']

auth= OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_tocken, access_secret)
api=tweepy.API(auth)

tweet_count = list()

for tweet in tweepy.Cursor(api.user_timeline,id='BDUTT',count=30 ).items():
    if tweet.text[:4] != 'RT @':
        rec = (tweet.text[:100], tweet.favorite_count, tweet.retweet_count)
        tweet_count.append(rec)
        print(rec)

pd.DataFrame(tweet_count).to_csv('bdutt.csv' , index=False)


from hamcrest import ( assert_that,
                       contains_exactly,
                       contains_inanyorder,
                       equal_to,
                       has_entries,
                       has_item,
                       matches_regexp )
from unittest import TestCase
import json
import requests

from ycmd.tests.javascript import setUpModule, tearDownModule # noqa
from ycmd.tests.javascript import IsolatedYcmd, PathToTestFile, SharedYcmd
from ycmd.tests.test_utils import ( BuildRequest,
                                    ChunkMatcher,
                                    CompletionEntryMatcher,
                                    LocationMatcher )
from ycmd.utils import ReadFile

def RunTest( app, test ):
  contents = ReadFile( test[ 'request' ][ 'filepath' ] )

  def CombineRequest( request, data ):
    kw = request
    request.update( data )
    return BuildRequest( **kw )

  app.post_json(
    '/event_notification',
    CombineRequest( test[ 'request' ], {
      'contents': contents,
      'filetype': 'javascript',
      'event_name': 'BufferVisit'
    } )
  )

  response = app.post_json(
    '/completions',
    CombineRequest( test[ 'request' ], {
      'contents': contents,
      'filetype': 'javascript',
      'force_semantic': True
    } )
  )

  print( 'completer response: ', json.dumps( response.json, indent = 2 ) )

  assert_that( response.status_code,
               equal_to( test[ 'expect' ][ 'response' ] ) )

  assert_that( response.json, test[ 'expect' ][ 'data' ] )

class GetCompletionsTest( TestCase ):
  @SharedYcmd
  def test_GetCompletions_Basic( self, app ):
    RunTest( app, {
      'description': 'Extra and detailed info when completions are methods',
      'request': {
        'line_num': 14,
        'column_num': 6,
        'filepath': PathToTestFile( 'test.js' )
      },
      'expect': {
        'response': requests.codes.ok,
        'data': has_entries( {
          'completions': contains_inanyorder(
            CompletionEntryMatcher(
              'methodA',
              '(method) Foo.methodA(): void',
              extra_params = {
                'kind': 'method',
                'detailed_info': '(method) Foo.methodA(): void\n\n'
                                 'Unicode string: 说话'
              }
            ),
            CompletionEntryMatcher(
              'methodB',
              '(method) Foo.methodB(): void',
              extra_params = {
                'kind': 'method',
                'detailed_info': '(method) Foo.methodB(): void'
              }
            ),
            CompletionEntryMatcher(
              'methodC',
              '(method) Foo.methodC(foo: any, bar: any): void',
              extra_params = {
                'kind': 'method',
                'detailed_info': '(method) Foo.methodC(foo: any, '
                                 'bar: any): void'
              }
            )
          )
        } )
      }
    } )

  @SharedYcmd
  def test_GetCompletions_Keyword( self, app ):
    RunTest( app, {
      'description': 'No extra and detailed info when completion is a keyword',
      'request': {
        'line_num': 1,
        'column_num': 5,
        'filepath': PathToTestFile( 'test.js' ),
      },
      'expect': {
        'response': requests.codes.ok,
        'data': has_entries( {
          'completions': has_item( {
            'insertion_text': 'class',
            'kind':           'keyword',
            'extra_data':     {}
          } )
        } )
      }
    } )

  @SharedYcmd
  def test_GetCompletions_AutoImport( self, app ):
    filepath = PathToTestFile( 'test.js' )
    RunTest( app, {
      'description': 'Symbol from external module can be completed and its '
                     'completion contains fixits to automatically import it',
      'request': {
        'line_num': 36,
        'column_num': 5,
        'filepath': filepath,
      },
      'expect': {
        'response': requests.codes.ok,
        'data': has_entries( {
          'completions': has_item( has_entries( {
            'insertion_text':  'Bår',
            'extra_menu_info': 'class Bår',
            'detailed_info':   'class Bår',
            'kind':            'class',
            'extra_data': has_entries( {
              'fixits': contains_inanyorder(
                has_entries( {
                  'text': 'Add import from "./unicode"',
                  'chunks': contains_exactly(
                    ChunkMatcher(
                      matches_regexp( '^import { Bår } from "./unicode";\r?\n'
                                      '\r?\n' ),
                      LocationMatcher( filepath, 1, 1 ),
                      LocationMatcher( filepath, 1, 1 )
                    )
                  ),
                  'location': LocationMatcher( filepath, 36, 5 )
                } )
              )
            } )
          } ) )
        } )
      }
    } )

  @IsolatedYcmd()
  def test_GetCompletions_IgnoreIdentifiers( self, app ):
    RunTest( app, {
      'description': 'Identifier "test" is not returned as a suggestion',
      'request': {
        'line_num': 5,
        'column_num': 6,
        'filepath': PathToTestFile( 'identifier', 'test.js' ),
      },
      'expect': {
        'response': requests.codes.ok,
        'data': has_entries( {
          'completions': contains_exactly(
            CompletionEntryMatcher(
              'foo',
              '(property) foo: string',
              extra_params = {
                'kind': 'property',
                'detailed_info': '(property) foo: string'
              }
            )
          )
        } )
      }
    } )


import numpy as np
import string
import num2words
import pandas as pd
from typing import List
import csv

MAX_LENGTH_SENTENCES = 10

filename_en = "es-en/europarl-v7.es-en.en"
english_sentences = []
with open(filename_en, 'r', encoding='UTF-8') as file:
    for line in file:
        english_sentences.append(line.rstrip())

filename_es = "es-en/europarl-v7.es-en.es"
spanish_sentences = []
with open(filename_es, 'r', encoding='UTF-8') as file:
    for line in file:
        spanish_sentences.append(line.rstrip())

nb_sentences_en = len(english_sentences)
nb_sentences_es = len(spanish_sentences)
print("nb en sentences : " + str(nb_sentences_en))
print("nb es sentences : " + str(nb_sentences_es))
print("nb tot sentences : " + str(nb_sentences_en + nb_sentences_es))

indexes = range(nb_sentences_en)
# to randomly select 10 percent of the data
# indexes_ten_percent = np.random.choice(indexes, int(nb_sentences_en / 10))
# randomly select 1 percent so the dataset is smaller
indexes_ten_percent = np.random.choice(indexes, int(nb_sentences_en / 100))
english_sentences = [english_sentences[i] for i in indexes_ten_percent]
spanish_sentences = [spanish_sentences[i] for i in indexes_ten_percent]

# pre-processing
english_sentences_preprocessed = []
spanish_sentences_preprocessed = []

def pre_process_list(list_sentences: list[str], list_preprocessed: list[str], language: str) -> list[str]:
    for curr_sentence in list_sentences:
        # lowercase
        curr_sentence = curr_sentence.lower()
        # remove punctuation
        curr_sentence = curr_sentence.translate(str.maketrans('', '', string.punctuation))
        # change numbers into word equivalents and remove white spaces
        curr_sentence = ' '.join([num2words.num2words(word, lang=language) if word.isdigit() else word
                                  for word in curr_sentence.split()])
        list_preprocessed.append(curr_sentence)
    return list_preprocessed

print(len(english_sentences))
print(len(spanish_sentences))
english_sentences_preprocessed = pre_process_list(english_sentences, english_sentences_preprocessed, "en")
spanish_sentences_preprocessed = pre_process_list(spanish_sentences, spanish_sentences_preprocessed, "es")

"""for i in range(10):
    print(english_sentences_preprocessed[i])
    print(spanish_sentences_preprocessed[i])

print(len(english_sentences_preprocessed))
print(len(spanish_sentences_preprocessed))"""
"""
headerList = ['english_sentences', 'spanish_stences']
with open('mixedtranslation.csv', 'w') as f:
    writer = csv.writer(f,fieldnames = headerList)
    writer.writerows(zip(english_sentences_preprocessed, spanish_sentences_preprocessed)) """

with open('mixedtranslation.csv', 'w') as csvfile:
    fieldnames = ['english_sentences', 'spanish_sentences']
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

    writer.writeheader()
    for i in range(len(english_sentences_preprocessed)):
        if len(english_sentences_preprocessed[i].split(" ")) <= MAX_LENGTH_SENTENCES and \
                len(spanish_sentences_preprocessed[i].split(" ")) <= MAX_LENGTH_SENTENCES:
            writer.writerow({'english_sentences': english_sentences_preprocessed[i],
                             'spanish_sentences': spanish_sentences_preprocessed[i]})


from flask import Flask, jsonify
import requests
import json

app = Flask(__name__)

recent = [] # Keep a local list of the 5 most recently retrieved users
# Reduces calls to external site reqres.in

def getUser(id):
    url = requests.get( "https://reqres.in/api/users/" + str(id) )
    userDict = json.loads(url.text) # Convert String to Python Dictionary
    return userDict["data"] # Only keep data related to the user

def checkRecent(id):
    global recent

    if not any(_["id"] == id for _ in recent):
        if len(recent) > 5:
            recent.pop(0) # Remove the oldest item from the recent list to make room for a new item

        recent.append(getUser(id)) # Add retrieved user to the recent list if not present

    return next(_ for _ in recent if _["id"] == id) # Send the user json from the recent list

@app.route("/user/<int:id>/", methods=['GET'])
def displayUserDetails(id):
    return checkRecent(id)

if __name__ == '__main__':
    app.run(debug = True, host = '0.0.0.0')


from django.db import models
from phonenumber_field.modelfields import PhoneNumberField
from stdimage.models import StdImageField
from django.utils.text import slugify
from django.contrib.auth.models import User
from django.contrib.postgres.fields import ArrayField
from knox.models import AuthToken

STATES = ['AC', 'AL', 'AP', 'AM', 'BA', 'CE', 'DF', 'ES', 'GO', 'MA', 'MT', 'MS', 'MG', 'PA',
    'PB', 'PR', 'PE', 'PI', 'RJ', 'RN', 'RS', 'RO', 'RR', 'SC', 'SP', 'SE', 'TO']

STATES_CHOICES = [(state, state) for state in STATES]

class Base(models.Model):
    created_at = models.DateField("Created", auto_now_add=True)
    updated_at = models.DateField("Updated", auto_now=True)
    is_active = models.BooleanField("Active", default=True)

    class Meta:
        abstract = True

class Market(Base):
    name = models.CharField("Name", max_length=100)
    logo = StdImageField('Logo', upload_to='logos', variations={'thumb': (124, 124)})
    website = models.URLField("Website", max_length=200)
    ecommerce = models.URLField("E-commerce", max_length=200)
    phone = PhoneNumberField('Phone Number', unique=True)
    address = models.CharField("Address", max_length=300)
    s3_scrapy_path = models.CharField("Scrapy data path", max_length=128, null=True, blank=True)

    class Meta:
        verbose_name = 'Market'
        verbose_name_plural = 'Markets'

    def __str__(self):
        return self.name

class Category(models.Model):
    name = models.CharField("Name", max_length=128)
    icon = models.CharField("Icon name", null=True, max_length=32)

    def __str__(self):
        return f"{self.name}"

class Product(Base):
    name = models.CharField("Name", max_length=128)
    slug = models.SlugField("Slug", max_length=128, allow_unicode=True, null=True, blank=True)
    image_url = models.URLField("Image", blank=True, null=True)
    market = models.ForeignKey(Market, on_delete=models.PROTECT, related_name="products")
    category = models.ForeignKey(Category, on_delete=models.PROTECT, related_name="products", null=True)
    department = models.CharField("Department", max_length=128, null=True, blank=True)
    sub_department = models.CharField("Sub-department", max_length=128, null=True, blank=True)
    tags = ArrayField(models.CharField('Tag', max_length=128), null=True, blank=True)
    price = models.DecimalField("Price", decimal_places=2, max_digits=10)
    is_available = models.BooleanField("Is available?", default=True)
    link = models.URLField("Link")
    description = models.TextField("Description", null=True, blank=True)
    code = models.IntegerField("Market Code", null=True, blank=True)
    is_offer = models.BooleanField("Is it on offer?", default=False)
    similar = models.ManyToManyField("Product", null=True, blank=True, related_name="similar_products")
    comparison_name = models.CharField("Comparison Name", max_length=128, null=True, blank=True)
    max_price = models.DecimalField("Max Price", decimal_places=2, max_digits=10, null=True, blank=True)
    min_price = models.DecimalField("Min Price", decimal_places=2, max_digits=10, null=True, blank=True)
    similar_to = models.ManyToManyField("Product", null=True, blank=True, related_name="similar_products_set", through="Similar")

    def __str__(self):
        return f"{self.name} - {self.market.name}"

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(Product, self).save(*args, **kwargs)

class Similar(models.Model):
    product1 = models.ForeignKey("Product", on_delete=models.CASCADE, related_name="product1")
    product2 = models.ForeignKey("Product", on_delete=models.CASCADE, related_name="product2")
    similarity = models.DecimalField("Max Price", decimal_places=2, max_digits=5, null=True, blank=True)

class ProductHistory(Base):
    product = models.ForeignKey(Product, on_delete=models.PROTECT, related_name="history")
    price = models.DecimalField("Price", decimal_places=2, max_digits=10)

class PriceAlert(Base):
    user = models.ForeignKey(User, on_delete=models.PROTECT, related_name="alerts")
    product = models.ForeignKey(Product, on_delete=models.PROTECT, related_name="alerts")
    expected_price = models.DecimalField("Expected Price", decimal_places=2, max_digits=10)
    was_reached = models.BooleanField("Price reached?", default=False)
    reached_date = models.DateField("Reached date", blank=True, null=True)
    reached_price = models.DecimalField("Reached Price", decimal_places=2, max_digits=10, blank=True, null=True)

class Favorite(Base):
    user = models.ForeignKey(User, on_delete=models.PROTECT, related_name="favorites_products")
    product = models.ForeignKey(Product, on_delete=models.PROTECT, related_name="favorites")

class Notification(Base):
    title = models.CharField("Notification Title", max_length=128)
    body = models.TextField("Notification Body")
    payload = models.JSONField("Notification Payload", null=True)
    user = models.ForeignKey(User, on_delete=models.PROTECT, related_name="notifications")
    was_sent = models.BooleanField("Notification sent?", default=False)

class Account(Base):
    user = models.OneToOneField(User, related_name="account", on_delete=models.PROTECT)
    phone = PhoneNumberField('Phone Number', null=True, blank=True)
    street = models.CharField("Street", max_length=128, null=True, blank=True)
    district = models.CharField("District", max_length=128, null=True, blank=True)
    address_number = models.IntegerField("Number", null=True, blank=True)
    zip_code = models.CharField("Zip Code", max_length=8, blank=True)
    city = models.CharField("City", max_length=128, null=True, blank=True)
    state = models.CharField("State", max_length=2, null=True, choices=STATES_CHOICES, blank=True)
    fcm_token = models.CharField("Firebase Token", max_length=255, blank=True, null=True)
    google_id = models.CharField("Google Account Id", max_length=128, null=True, blank=True)


from rest_framework import serializers
from .models import ProductInfo

class ProductSerializer(serializers.ModelSerializer):
    class Meta:
        model = ProductInfo
        fields = [
            'code',
            'name',
            'model',
            'createtime',
            'modifytime',
        ]
        read_only_fields = [
            'createtime',
        ]
        extra_kwargs = {"type": {"write_only": True}}


from __future__ import absolute_import
from __future__ import unicode_literals

import logging

import docker
import pytest

from compose import container
from compose.cli.errors import UserError
from compose.cli.formatter import ConsoleWarningFormatter
from compose.cli.main import call_docker
from compose.cli.main import convergence_strategy_from_opts
from compose.cli.main import filter_containers_to_service_names
from compose.cli.main import setup_console_handler
from compose.cli.main import warn_for_swarm_mode
from compose.service import ConvergenceStrategy
from tests import mock

def mock_container(service, number):
    return mock.create_autospec(
        container.Container,
        service=service,
        number=number,
        name_without_project='{0}_{1}'.format(service, number))

@pytest.fixture
def logging_handler():
    stream = mock.Mock()
    stream.isatty.return_value = True
    return logging.StreamHandler(stream=stream)

class TestCLIMainTestCase(object):

    def test_filter_containers_to_service_names(self):
        containers = [
            mock_container('web', 1),
            mock_container('web', 2),
            mock_container('db', 1),
            mock_container('other', 1),
            mock_container('another', 1),
        ]
        service_names = ['web', 'db']
        actual = filter_containers_to_service_names(containers, service_names)
        assert actual == containers[:3]

    def test_filter_containers_to_service_names_all(self):
        containers = [
            mock_container('web', 1),
            mock_container('db', 1),
            mock_container('other', 1),
        ]
        service_names = []
        actual = filter_containers_to_service_names(containers, service_names)
        assert actual == containers

    def test_warning_in_swarm_mode(self):
        mock_client = mock.create_autospec(docker.APIClient)
        mock_client.info.return_value = {'Swarm': {'LocalNodeState': 'active'}}

        with mock.patch('compose.cli.main.log') as fake_log:
            warn_for_swarm_mode(mock_client)
            assert fake_log.warn.call_count == 1

class TestSetupConsoleHandlerTestCase(object):

    def test_with_tty_verbose(self, logging_handler):
        setup_console_handler(logging_handler, True)
        assert type(logging_handler.formatter) == ConsoleWarningFormatter
        assert '%(name)s' in logging_handler.formatter._fmt
        assert '%(funcName)s' in logging_handler.formatter._fmt

    def test_with_tty_not_verbose(self, logging_handler):
        setup_console_handler(logging_handler, False)
        assert type(logging_handler.formatter) == ConsoleWarningFormatter
        assert '%(name)s' not in logging_handler.formatter._fmt
        assert '%(funcName)s' not in logging_handler.formatter._fmt

    def test_with_not_a_tty(self, logging_handler):
        logging_handler.stream.isatty.return_value = False
        setup_console_handler(logging_handler, False)
        assert type(logging_handler.formatter) == logging.Formatter

class TestConvergeStrategyFromOptsTestCase(object):

    def test_invalid_opts(self):
        options = {'--force-recreate': True, '--no-recreate': True}
        with pytest.raises(UserError):
            convergence_strategy_from_opts(options)

    def test_always(self):
        options = {'--force-recreate': True, '--no-recreate': False}
        assert (
            convergence_strategy_from_opts(options) ==
            ConvergenceStrategy.always
        )

    def test_never(self):
        options = {'--force-recreate': False, '--no-recreate': True}
        assert (
            convergence_strategy_from_opts(options) ==
            ConvergenceStrategy.never
        )

    def test_changed(self):
        options = {'--force-recreate': False, '--no-recreate': False}
        assert (
            convergence_strategy_from_opts(options) ==
            ConvergenceStrategy.changed
        )

def mock_find_executable(exe):
    return exe

@mock.patch('compose.cli.main.find_executable', mock_find_executable)
class TestCallDocker(object):
    def test_simple_no_options(self):
        with mock.patch('subprocess.call') as fake_call:
            call_docker(['ps'], {})

        assert fake_call.call_args[0][0] == ['docker', 'ps']

    def test_simple_tls_option(self):
        with mock.patch('subprocess.call') as fake_call:
            call_docker(['ps'], {'--tls': True})

        assert fake_call.call_args[0][0] == ['docker', '--tls', 'ps']

    def test_advanced_tls_options(self):
        with mock.patch('subprocess.call') as fake_call:
            call_docker(['ps'], {
                '--tls': True,
                '--tlscacert': './ca.pem',
                '--tlscert': './cert.pem',
                '--tlskey': './key.pem',
            })

        assert fake_call.call_args[0][0] == [
            'docker', '--tls', '--tlscacert', './ca.pem', '--tlscert',
            './cert.pem', '--tlskey', './key.pem', 'ps'
        ]

    def test_with_host_option(self):
        with mock.patch('subprocess.call') as fake_call:
            call_docker(['ps'], {'--host': 'tcp://mydocker.net:2333'})

        assert fake_call.call_args[0][0] == [
            'docker', '--host', 'tcp://mydocker.net:2333', 'ps'
        ]

    def test_with_http_host(self):
        with mock.patch('subprocess.call') as fake_call:
            call_docker(['ps'], {'--host': 'http://mydocker.net:2333'})

        assert fake_call.call_args[0][0] == [
            'docker', '--host', 'tcp://mydocker.net:2333', 'ps',
        ]

    def test_with_host_option_shorthand_equal(self):
        with mock.patch('subprocess.call') as fake_call:
            call_docker(['ps'], {'--host': '=tcp://mydocker.net:2333'})

        assert fake_call.call_args[0][0] == [
            'docker', '--host', 'tcp://mydocker.net:2333', 'ps'
        ]


import argparse
import time
from datetime import datetime
from typing import TypeVar

import faiss
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
from torch.nn.functional import cosine_similarity

from common.utils import top_k_unsorted

Tensor = TypeVar('torch.Tensor')

def compute_runtime_faiss(pool: Tensor, queries: Tensor, K: int, device: torch.device) -> time:
    assert(pool.shape[-1] == queries.shape[-1]), "Error! Pool and queries do not have the same dimensions!"
    # Create FAISS index and load data.
    dim = pool.shape[-1]
    pool = pool.squeeze(1)
    pool = pool.numpy().astype(np.float32)
    queries = queries.numpy().astype(np.float32)
    index = faiss.IndexFlatL2(dim)   # build the index.
    index.add(pool)                  # add vectors to the index.

    start = time.time()
    for q in queries:
        D, I = index.search(q, K)     # Find top K nearest neighbourhs of q.  I= indexes of top K, D= distances of the top K
        I = I.squeeze()
        import pdb; pdb.set_trace()
    end = time.time()
    return (end-start) / len(queries)

def compute_runtime_cosine(
    pool: Tensor,
    queries: Tensor,
    K: int,
    device: torch.device) -> time:
    queries, pool = queries.double(), pool.double()
    nq        = queries.shape[0]
    start     = time.time()
    for query in queries:
        sim = cosine_similarity(query, pool).unsqueeze(0)
        top_k_unsorted(sim, k=K, descending=True)
    end       = time.time()
    return (end - start) / len(queries)

def scan_parameters(pool: Tensor, queries: Tensor) -> pd.DataFrame:
    pool = pool.repeat(200, 1)  # Increase numper of pools to 1M
    queries = queries[: 1000]  # Decrease number of queries to 1K

    K_size = [10, 100, 200, 400, 600, 800, 1000]
    pool_size = [
        2_000, 5_000, 10_000, 20_000, 30_000, 40_000, 50_000, 100_000, 1_000_000
    ]

    res = []
    # Scan runtime vs pool size, fixing topK.
    topK = 800
    for size in pool_size:
        t_cos = compute_runtime_cosine(pool[: size], queries, topK, device)
        t_faiss = compute_runtime_faiss(pool[: size], queries, topK, device)
        print('Runtime: size: {} K:{}   --   cosine: {}     faiss:{}'.format(size, topK, t_cos, t_faiss))
        res.append((size, topK, t_cos, t_faiss))
    # Scan runtime vs top K size, fixing pool size.
    size = 1_000_000
    for topK in K_size:
        t_cos = compute_runtime_cosine(pool[: size], queries, topK, device)
        t_faiss = compute_runtime_faiss(pool[: size], queries, topK, device)
        print('Runtime: size: {} K:{}   --   cosine: {}     faiss:{}'.format(size, topK, t_cos, t_faiss))
        res.append((size, topK, t_cos, t_faiss))

    res = pd.DataFrame(res, columns=['db_size', 'top K', 'cosine', 'faiss'])
    return res

def plot_results(data: pd.DataFrame) -> None:
    ax = plt.gca()
    df = data.iloc[: 9]
    df.plot(kind='line',x='db_size',y='cosine', color='blue', ax=ax)
    df.plot(kind='line',x='db_size',y='faiss', color='red', ax=ax)
    ax.set_title('Top K retrieval time VS pool size\n(K=800)')
    ax.set_xlabel('pool size')
    ax.set_ylabel('time (sec)')
    plt.savefig('evaluate/results/runtime_vs_poolsize.png')
    plt.close()
    ax = plt.gca()
    df = data.iloc[9: ]
    df.plot(kind='line',x='top K',y='cosine', color='blue', ax=ax)
    df.plot(kind='line',x='top K',y='faiss', color='red', ax=ax)
    ax.set_title('Top K retrieval time VS K\n(pool size=1M)')
    ax.set_xlabel('top K nearest neighbours')
    ax.set_ylabel('time (sec)')
    plt.savefig('evaluate/results/runtime_vs_K.png')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--coco_visual',
        type=str,
        default='cache/coco_visual_cache.pth',
        help='Path to images tensor pool'
    )
    parser.add_argument(
        '--coco_text',
        type=str,
        default='cache/coco_text_cache.pth',
        help='Path to text tensor pool'
    )
    parser.add_argument(
        '--cuda',
        action='store_true',
        default=False
    )
    args = parser.parse_args()

    with open(args.coco_visual, 'rb') as fp:
        pool  = torch.load(fp)
    with open(args.coco_text, 'rb') as fp:
        queries  = torch.load(fp)
    device = torch.device('cuda:0' if args.cuda else 'cpu')

    print(args)

    pool      = torch.stack([v for v in pool.values()]).squeeze(1).to(device)
    queries   = torch.stack([v for v in queries.values()]).to(device)

    torch.set_num_threads(1)
    res = scan_parameters(pool, queries)
    res.to_csv('evaluate/results/exec_runtimes_scan.csv')
    # res = pd.read_csv('evaluate/results/exec_runtimes_scan.csv')
    plot_results(res)