True

from collections import OrderedDict

TOKEN_INT = b'i'
TOKEN_LIST = b'l'
TOKEN_DICT = b'd'
TOKEN_END = b'e'
TOKEN_STR_SEPARATOR = b':'

class Decoder:
    def __init__(self, data):
        self._data = data
        self._idx = 0

    def _is_out_of_range(self, idx: int):
        return idx >= len(self._data)

    def _peek(self):
        """
        Return current char from data or None if eol
        """
        return None if self._is_out_of_range(self._idx + 1) \
            else self._data[self._idx: self._idx+1]

    def _read_next(self, length: int):
        if self._idx + length > len(self._data):
            raise IndexError()
        res = self._data[self._idx: self._idx + length]
        self._idx += length
        return res

    def _read_until(self, token: bytes):
        try:
            occurrence = self._data.index(token, self._idx)
            res = self._data[self._idx: occurrence]
            self._idx = occurrence + 1
            return res
        except ValueError:
            raise RuntimeError("Can't find token {}.".format(str(token)))

    def reset(self):
        self._idx = 0

    def parse(self):
        c = self._peek()
        if c is None:
            raise EOFError("Unexpected EOF")
        if c == TOKEN_END:
            return None
        if c in b'0123456789':
            return self.parse_string()
        if c == TOKEN_INT:
            return self.parse_int()
        if c == TOKEN_LIST:
            return self.parse_list()
        if c == TOKEN_DICT:
            return self.parse_dict()
        else:
            raise RuntimeError("Unknown token \"{}\" got at {}".format(
                c, self._idx)
            )

    def parse_string(self):
        bytes_to_read = int(self._read_until(TOKEN_STR_SEPARATOR))
        return self._read_next(bytes_to_read)

    def parse_int(self):
        self._idx += 1 # Token
        return int(self._read_until(TOKEN_END))

    def parse_list(self):
        self._idx += 1 # Token
        res = list()
        while self._data[self._idx: self._idx + 1] != TOKEN_END:
            res.append(self.parse())
        self._idx += 1 # END tocken
        return res

    def parse_dict(self):
        self._idx += 1 # Token
        res = dict()
        while self._data[self._idx: self._idx + 1] != TOKEN_END:
            key = self.parse_string()
            obj = self.parse()
            res[key] = obj
        self._idx += 1 # END tocken
        return res

class Encoder:
    @staticmethod
    def encode(data):
        if type(data) == str:
            return Encoder.encode_strint(data)
        if type(data) == int:
            return Encoder.encode_int(data)
        if type(data) == list:
            return Encoder.encode_list(data)
        if type(data) == dict or type(data) == OrderedDict:
            return Encoder.encode_dict(data)
        if type(data) == bytes:
            return Encoder.encode_bytes(data)
        raise TypeError("Got unknown type: {}.".format(type(data)))

    @staticmethod
    def encode_strint(data: str):
        return bytes("{}:{}".format(len(data), data), "UTF-8")

    @staticmethod
    def encode_int(data: int):
        return bytes("i{}e".format(data), "UTF-8")

    @staticmethod
    def encode_list(data: list):
        res = bytearray('l', 'utf-8')
        res += b''.join([Encoder.encode(item) for item in data])
        res += b'e'
        return res

    @staticmethod
    def encode_dict(data: dict or OrderedDict):
        res = bytearray('d', 'utf-8')
        for k, v in data.items():
            key = Encoder.encode(k)
            value = Encoder.encode(v)
            if key and value:
                res += key
                res += value
            else:
                raise RuntimeError('Bad dict')
        res += b'e'
        return res

    @staticmethod
    def encode_bytes(data: bytes):
        return str.encode(str(len(data))) + b":" + data


from sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score, f1_score, log_loss
from sklearn.metrics import mean_squared_error,r2_score
from sklearn.metrics import confusion_matrix, classification_report
import numpy as np

# https://scikit-learn.org/stable/modules/model_evaluation.html

def clf_metrics(y, y_pred):
    """
    caculate classification machines performance

    Args:
    ----------
        y : y actual
        y_pred :  y predict
    Return:
    ----------
        d : Train varable
    """
    d={}
    d["accuracy"] = round(accuracy_score(y, y_pred),4)
    d["precision"] = round(precision_score(y, y_pred),4)
    d["recall"] = round(recall_score(y, y_pred),4)
    d["f1"] = round(f1_score(y, y_pred),4)
    d["AUC"] = round(roc_auc_score(y, y_pred),4)
    d["log_loss"] = round(log_loss(y, y_pred),4)
    return d

def reg_metrics(y, y_pred):
    """
    caculate regression machines performance

    Args:
    ----------
        y : y actual
        y_pred :  y predict
    Return:
    ----------
        d : Train varable
    """
    d={}
    d["MSE"] = round(mean_squared_error(y,y_pred),4)
    d["RMSE"] = round(np.sqrt(mean_squared_error(y, y_pred)),4)
    d["R2"] = round(r2_score(y,y_pred),4)
    return d

def clf_report(y, y_pred):
    """
    caculate classification machines report

    Args:
    ----------
        y : y actual
        y_pred :  y predict
    Return:
    ----------
        confusionmatrix : confusion matrix
        report : some metrics report
    """
    confusionmatrix = confusion_matrix(y,y_pred)
    report = classification_report(y, y_pred)
    print("Confusion Matrix")
    print(confusionmatrix)
    print()
    print("Classification Report")
    print(report)
    #return confusionmatrix ,report


from unittest import TestCase
from boris.forecast import WeatherForecaster, Peak, Weather
from boris.planets import SolarSystem
from mock import Mock
from hamcrest import assert_that, is_, has_length, has_properties, calling, \
    raises
from dateutil.relativedelta import relativedelta

class WeatherForecasterTest(TestCase):

    def setUp(self):
        self.solar_system = SolarSystem()
        self.forecaster = WeatherForecaster(self.solar_system)

    def test_it_forecasts_drought_if_planets_are_aligned_with_sun(self):
        self._mock_solar_system_state(aligned_with_sun=True)
        assert_that(self.forecaster.current_weather(), is_(Weather.DROUGHT))

    def test_it_forecasts_optimal_conditions_if_planets_are_aligned(self):
        self._mock_solar_system_state(aligned=True)
        assert_that(self.forecaster.current_weather(), is_(Weather.OPTIMAL))

    def test_it_forecasts_rain_if_planets_polygon_contains_sun(self):
        self._mock_solar_system_state(contains_sun=True)
        assert_that(self.forecaster.current_weather(), is_(Weather.RAINY))

    def test_it_forecasts_regular_wetaher_otherwise(self):
        self._mock_solar_system_state(
            aligned_with_sun=False, aligned=False,  contains_sun=False)
        assert_that(self.forecaster.current_weather(), is_(Weather.REGULAR))

    def test_it_generates_a_forecast_for_a_given_time_span(self):
        self.forecaster.current_weather = Mock()
        self.forecaster.current_weather.side_effect = [
            Weather.DROUGHT, Weather.RAINY, Weather.RAINY]
        self.solar_system.planets_perimeter = Mock()
        self.solar_system.planets_perimeter.side_effect = [100, 200]
        forecast = self.forecaster.forecast_for(relativedelta(days=3))
        assert_that(forecast.periods, has_length(2))
        assert_that(forecast.periods[0], has_properties({
            'weather': Weather.DROUGHT, 'start_day': 0,
            'end_day': 0, 'peak': None
        }))
        assert_that(forecast.periods[1], has_properties({
            'weather': Weather.RAINY, 'start_day': 1,
            'end_day': 2, 'peak': Peak(2, 200)
        }))

    def test_it_fails_if_period_is_less_than_a_day(self):
        assert_that(calling(self.forecaster.forecast_for)
                    .with_args(relativedelta(hours=23)),
                    raises(ValueError, 'Invalid period.*'))

    def _mock_solar_system_state(self, aligned_with_sun=False, aligned=False,
                                 contains_sun=False):
        self.solar_system.are_planets_aligned_with_sun = \
            Mock(return_value=aligned_with_sun)
        self.solar_system.are_planets_aligned = \
            Mock(return_value=aligned)
        self.solar_system.does_planets_polygon_contain_sun = \
            Mock(return_value=contains_sun)


import csv
import cv2
import numpy as np
import os
import math
from random import shuffle

samples = []
with open('data/driving_log.csv') as csvfile:
	reader = csv.reader(csvfile)
	for line in reader:
		samples.append(line)

recover_samples = []
with open('data_recover/driving_log.csv') as csvfile:
	reader = csv.reader(csvfile)
	for line in reader:
		recover_samples.append(line)

shuffle(samples)
shuffle(recover_samples)

image_paths = []
angles = []
for sample in samples:
    name = 'data/IMG/'+ sample[0].split('/')[-1]
    image_paths.append(name)
    center_angle = float(sample[3])
    angles.append(center_angle)

for sample in recover_samples:
    name = 'data_recover/IMG/'+ sample[0].split('/')[-1]
    image_paths.append(name)
    center_angle = float(sample[3])
    angles.append(center_angle)

print("size of angles:", len(angles))
print("size of image_paths", len(image_paths))
print("angles[14]", angles[14])
print("image_paths[14]", image_paths[14])
image = cv2.imread(image_paths[14])
cv2.imshow("im1", image)
cv2.waitKey(0)


# -*- coding: utf-8 -*-
"""
__title__="train"
__author__="ngc7293"
__mtime__="2020/9/20"
"""
from data_load import get_dataloader
from model import TextCnn, TextRnn
import numpy as np
import pandas as pd

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import optim

if __name__ == '__main__':
    lr = 0.001
    epoches = 50
    class_num = 5
    batch_size = 32
    sentence_len = 40
    embedding_dim = 100

    device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')

    train_loader, test_loader, embed_pretrain_glove, vocab_size = get_dataloader(sentence_len=sentence_len, pre_model="glove")

    _, _, embed_pretrain_word2vec, _ = get_dataloader(sentence_len=sentence_len, pre_model="word2vec")

    rnn_model_with_glove = TextRnn(vocab_size=vocab_size, embedding_dim=embedding_dim, class_num=5, hidden_size=128,
                                   embedding_pretrained=embed_pretrain_glove, device=device)

    cnn_model_with_glove = TextCnn(vocab_size=vocab_size, embedding_dim=embedding_dim, hidden_size=128, class_num=class_num,
                                   embedding_pretrained=embed_pretrain_glove)

    rnn_model_without = TextRnn(vocab_size=vocab_size, embedding_dim=embedding_dim, class_num=5, hidden_size=128,
                                embedding_pretrained=None, device=device)

    cnn_model_without = TextCnn(vocab_size=vocab_size, embedding_dim=embedding_dim, hidden_size=128, class_num=class_num,
                                embedding_pretrained=None)

    rnn_model_with_word2vec = TextRnn(vocab_size=vocab_size, embedding_dim=embedding_dim, class_num=5, hidden_size=128,
                                embedding_pretrained=embed_pretrain_word2vec, device=device)

    cnn_model_with_word2vec =  TextCnn(vocab_size=vocab_size, embedding_dim=embedding_dim, hidden_size=128, class_num=class_num,
                                embedding_pretrained=embed_pretrain_word2vec)

    model_list = [rnn_model_with_glove,
                  rnn_model_without,
                  rnn_model_with_word2vec,
                  cnn_model_with_glove,
                  cnn_model_without,
                  cnn_model_with_word2vec]

    model_type_list = [
        "rnn with glove",
        "rnn with random wmbedding",
        "rnn with word2vec",
        "cnn with glove",
        "cnn with random embedding",
        "cnn with word2vec"
    ]
    # model_list = [rnn_model_with_word2vec]

    loss_history_list = []
    train_acc_list = []
    test_acc_list = []

    for model,model_type in zip(model_list,model_type_list):

        # if model_type == "cnn": model = TextCnn(vocab_size=vocab_size, embedding_dim=50, hidden_size=128,
        # class_num=class_num,  embedding_pretrained=embed_pretrain)

        print("traing %s model ..."%model_type)
        optimizer = optim.Adam(model.parameters(), lr=lr)
        loss_function = nn.CrossEntropyLoss()

        loss_history = []
        train_acc_history = []
        test_acc_history = []

        model.to(device)
        for epoch in range(epoches):
            model.train()
            losses = []
            count = 0
            for idx, (x, y) in enumerate(train_loader):
                x, y = x.to(device), y.to(device)
                # if len(x) < batch_size:
                #     break
                pred = model(x)
                optimizer.zero_grad()
                loss = loss_function(pred, y)
                loss.backward()
                losses.append(loss.item())
                optimizer.step()

            train_loss = np.mean(losses)
            loss_history.append(train_loss)

            # model.eval() will notify all your layers that you are in eval mode, that way, batchnorm or dropout
            # layers will work in eval mode instead of training mode. torch.no_grad() impacts the autograd engine and
            # deactivate it. It will reduce memory usage and speed up computations but you won’t be able to backprop
            # (which you don’t want in an eval script).

            model.eval()

            train_acces = []

            for idx, (x, y) in enumerate(train_loader):
                x = x.to(device)
                y = y.numpy()
                y_pred = np.argmax(model(x).cpu().detach().numpy(), 1)

                acc = np.mean(y_pred == y, dtype=np.float)
                train_acces.append(acc)
            train_acc = np.mean(train_acces)
            train_acc_history.append(train_acces)

            test_acces = []
            for idx, (x, y) in enumerate(test_loader):
                x = x.to(device)
                y = y.numpy()
                y_pred = np.argmax(model(x).cpu().detach().numpy(), 1)

                acc = np.mean(y_pred == y, dtype=np.float)
                test_acces.append(acc)

            test_acc = np.mean(test_acces)
            test_acc_history.append(test_acces)

            print("epoch %d train loss:%.6f,train acc:%.6f test acc:%.7f" % (
            epoch, float(train_loss), float(train_acc), float(test_acc)))

        loss_history_list.append(loss_history)
        train_acc_list.append(train_acc_history)
        test_acc_list.append(test_acc_history)

    pd.DataFrame(loss_history_list).to_csv("./result/loss_history_100d.csv")
    pd.DataFrame(train_acc_list).to_csv("./result/acc_history_100d.csv")
    pd.DataFrame(test_acc_list).to_csv("./result/test_acc_history_100d.csv")


from django.shortcuts import render, redirect

import datetime
from django.utils import timezone

from .. import calgen
from .. import taskops

def cal_monthly(request, dest_year='', dest_month=''):
	# form handling
	if request.method == 'POST':
		print("wrong access")

	# choose month
	if dest_year == '':
		base_day = timezone.now()
	else:
		base_day = datetime.datetime(int(dest_year), int(dest_month), 1)

	context = calgen.populate_cal_monthly(base_day)

	# add form
	context.update(taskops.populate_task_form())

	return render(request, 'tdcs/calmonthly.html', context)

def cal_weekly(request, dest_year='', dest_month=''):
	# form handling
	if request.method == 'POST':
		print("wrong access")

	if dest_year == '':
		base_day = timezone.now()
	else:
		base_day = datetime.datetime(int(dest_year), int(dest_month), 1)

	context = calgen.populate_cal_weekly(base_day)

	# add form
	context.update(taskops.populate_task_form())

	return render(request, 'tdcs/calweekly.html', context)

def cal_daily(request):
	return render(request, 'tdcs/caldaily.html')

def create_task(request):
	if request.method == 'POST':
		dest = taskops.create_task_from_form(request)

	return redirect('/tdcs/cal/'+str(dest.year)+'/'+str(dest.month))

def edit_task(request):
	if request.method == 'POST':
		if 'edittaskbtn' in request.POST:
			dest = taskops.edit_task_from_form(request)
		elif 'completetaskbtn' in request.POST:
			dest = taskops.complete_task_from_form(request)
		elif 'deletetaskbtn' in request.POST:
			dest = taskops.delete_task_from_form(request)

	return redirect('/tdcs/cal/'+str(dest.year)+'/'+str(dest.month))


#!/usr/bin/python3

# The MIT License (MIT)
#
# Copyright (c) 2020 Susam Pal
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Scrape https://www.mohfw.gov.in/ to retrieve COVID-19 statistics.

To run this scraper and print a summary of the scraped data, enter this
command at the top-level directory of this project:

    python3 -m py.mohfw
"""

import datetime
import json
import re
import sys
import urllib
import urllib.request

from py import log

class Data:
    def __init__(self):
        """Container for all data read and derived from MoHFW."""
        self.total = -1
        self.active = -1
        self.cured = -1
        self.death = -1
        self.migrated = -1
        self.ref_datetime = None
        self.ref_date = ''
        self.ref_time = ''
        self.foreign = -1
        self.regions = {}
        self.regions_total = -1
        self.regions_cured = -1
        self.regions_death = -1
        self.regions_active = -1

def load_home_data():
    """Return data retrieved from MoHFW website."""
    data = Data()

    # Save the response from MoHFW as a list of lines.
    url = 'https://www.mohfw.gov.in/'
    log.log('Connecting to {} ...', url)
    response = urllib.request.urlopen(url).read().decode('utf-8')
    lines = [l.strip() for l in response.splitlines()]
    lines = [l for l in lines if l != '']

    # Parsers.
    strong_re = re.compile(r'.*(.*)')
    time_re = re.compile(r'.*as on\s*:\s*(\d.*) IST')
    foreign_re = re.compile(r'.*[Ii]ncluding (\d+)? ?[Ff]oreign')
    td_re = re.compile(r'.*([^#]*).*	')
    parser_state = 'DEFAULT'

    # Parse the response.
    for i, line in enumerate(lines):
        if data.active == -1 and 'Active Cases' in line:
            data.active = int(strong_re.match(lines[i - 1]).group(1))
        elif data.cured == -1 and 'Cured' in line:
            data.cured = int(strong_re.match(lines[i - 1]).group(1))
        elif data.death == -1 and 'Deaths' in line:
            data.death = int(strong_re.match(lines[i - 1]).group(1))
        elif data.migrated == -1 and 'Migrated' in line:
            data.migrated = int(strong_re.match(lines[i - 1]).group(1))
        elif data.ref_datetime == None and 'as on' in line:
            t = time_re.match(line).group(1)
            data.ref_datetime = datetime.datetime.strptime(t, '%d %B %Y, %H:%M')
            data.ref_date = data.ref_datetime.strftime('%Y-%m-%d')
            data.ref_time = data.ref_datetime.strftime('%H:%M')
        elif data.foreign == -1 and 'foreign' in line:
            n = foreign_re.match(line).group(1)
            if n is not None:
                data.foreign = int(n)
        elif '
' in line:
            parser_state = 'REGION'
        elif parser_state == 'REGION' and '
' in line:
            if 'Total' in lines[i + 2]:
                parser_state = 'REGION_TOTAL'
                continue
            # Parse
            region_name = td_re.match(lines[i + 2]).group(1)
            active = td_re.match(lines[i + 3]).group(1)
            cured = td_re.match(lines[i + 4]).group(1)
            death = td_re.match(lines[i + 5]).group(1)
            total = td_re.match(lines[i + 6]).group(1)
            # Normalize
            if region_name.startswith('Cases being reassigned'):
                region_name = 'reassigned'
            total = int(total) if total else -1
            cured = int(cured) if cured else -1
            death = int(death) if death else -1
            active = int(active) if active else -1
            # Save
            data.regions[region_name] = (total, active, cured, death)
        elif parser_state == 'REGION_TOTAL' and 'Total' in line:
            parser_state = 'DEFAULT'
            data.regions_active = int(strong_re.match(lines[i + 1]).group(1))
            data.regions_cured = int(strong_re.match(lines[i + 3]).group(1))
            data.regions_death = int(strong_re.match(lines[i + 6]).group(1))
            data.regions_total = int(strong_re.match(lines[i + 9]).group(1))

    data.total = data.active + data.cured + data.death + data.migrated

    # Validations.
    if data.total != data.regions_total:
        log.log('home page: Mismatch in total and regions_total')
    if data.active != data.regions_active:
        log.log('home page: Mismatch in active and regions_active')
    if data.cured + data.migrated != data.regions_cured:
        log.log('home page: Mismatch in cured + migrated and regions_cured')
    if data.death != data.regions_death:
        log.log('home page: Mismatch in death and regions_death')
    return data

def load_dash_data():
    """Return data retrieved from MoHFW dashboard page."""
    data = Data()

    # Retrieve MoHFW dashboard HTML.
    url = 'https://www.mohfw.gov.in/index.php'
    log.log('Connecting to {} ...', url)
    response = urllib.request.urlopen(url).read().decode('utf-8')
    lines = [l.strip() for l in response.splitlines()]
    lines = [l for l in lines if l != '']

    # Parsers.
    strong_re = re.compile(r'.*(.*)')
    time_re = re.compile(r'.*as on\s*:\s*(\d.*) IST')
    js_re = re.compile(r"\['(.*)', (.*), (.*), (.*)\],")
    parser_state = 'DEFAULT'

    # Parse the response.
    for i, line in enumerate(lines):
        if data.active == -1 and 'Active Cases' in line:
            data.active = int(strong_re.match(lines[i - 1]).group(1))
        elif data.cured == -1 and 'Cured' in line:
            data.cured = int(strong_re.match(lines[i - 1]).group(1))
        elif data.death == -1 and 'Deaths' in line:
            data.death = int(strong_re.match(lines[i - 1]).group(1))
        elif data.migrated == -1 and 'Migrated' in line:
            data.migrated = int(strong_re.match(lines[i - 1]).group(1))
        elif data.ref_datetime == None and 'as on' in line:
            t = time_re.match(line).group(1)
            data.ref_datetime = datetime.datetime.strptime(t, '%d %B %Y, %H:%M')
            data.ref_date = data.ref_datetime.strftime('%Y-%m-%d')
            data.ref_time = data.ref_datetime.strftime('%H:%M')
        elif 'Hover' in line:
            break

    data.total = data.active + data.cured + data.death + data.migrated

    # Retrieve MoHFW JSON data.
    url = 'https://www.mohfw.gov.in/data/data.json'
    log.log('Connecting to {} ...', url)
    items = json.load(urllib.request.urlopen(url))

    # Parse the response.
    for item in items:
        region_name = item['state_name']
        total = int(item['positive'])
        cured = int(item['cured'])
        death = int(item['death'])
        active = total - cured - death
        data.regions[region_name] = (total, active, cured, death)

    # Region totals.
    data.regions_total = sum(v[0] for v in data.regions.values())
    data.regions_active = sum(v[1] for v in data.regions.values())
    data.regions_cured = sum(v[2] for v in data.regions.values())
    data.regions_death = sum(v[3] for v in data.regions.values())

    # Validations.
    if data.total != data.regions_total:
        log.log('dashboard: Mismatch in total and regions_total')
    if data.active != data.regions_active:
        log.log('dashboard: Mismatch in active and regions_active')
    if data.cured + data.migrated != data.regions_cured:
        log.log('dashboard: Mismatch in cured + migrated and regions_cured')
    if data.death != data.regions_death:
        log.log('dashboard: Mismatch in death and regions_death')

    return data

def make_summary(data):
    """Print summary of data on the terminal."""
    out = []
    out.append('ref_datetime: {}'.format(data.ref_datetime))
    out.append('overall: total: {}; active: {}; cured: {}; death: {}; '
               'migrated: {}'.format(data.total, data.active,
                                     data.cured, data.death, data.migrated))
    out.append('regions: total: {}; active: {}; cured: {}; death: {}; '
               'foreign: {}'.format(data.regions_total, data.regions_active,
                                    data.regions_cured, data.regions_death,
                                    data.foreign))
    out.append('')
    out.append('regions: {}'.format(sorted(data.regions.items())))
    out.append('')
    return '\n'.join(out)

def make_json_entry(data):
    """Return JSON entry to be added to indiacovid19.json."""
    return ('  [ "{}",  {:5},  {:5},  {:5},  {:5},  "{} {}",  '
            '"https://indiacovid19.github.io/webarchive/mohfw/{}_{}/",'
            '      "" ]'
            .format(data.ref_date, data.active, data.cured, data.death,
                    data.migrated, data.ref_date, data.ref_time,
                    data.ref_date, data.ref_time.replace(':', '')))

def update_json(json_entry):
    """Update indiacovid19.json with the specified JSON entry."""
    with open('indiacovid19.json') as f:
        j = f.read()
    if json_entry in j:
        print('JSON archive is up-to-date')
        return
    print('Updating JSON archive ...')
    lines = j.splitlines()
    for i, line in enumerate(lines):
        if 1 < i < len(lines) - 1 and line.endswith(']'):
            lines[i] = line + ','
    lines[len(lines) - 1] = json_entry
    lines.append(']')
    output = '\n'.join(lines) + '\n'
    with open('indiacovid19.json', 'w') as f:
        f.write(output)
    print('Done')

def print_summary(data, heading):
    """Print a summary of the retrieved data on console."""
    print(heading)
    print('-' * len(heading))
    print(make_summary(data))
    json_entry = make_json_entry(data)
    print('JSON octuple for indiacovid19.json:')
    print()
    print(json_entry)
    print()
    update_json(json_entry)
    print()

def main():
    home_data = load_home_data()
    #dash_data = load_dash_data()
    print()
    print_summary(home_data, 'HOME PAGE DATA')
    #print_summary(dash_data, 'DASHBOARD DATA')

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
import sqlite3
import yaml
import os
import re
from datetime import timedelta, datetime

def update_db(db_name, sw_src_file=None, dhcp_src_file=None):
    if not os.path.exists(db_name):
        print('База данных не существует. Перед добавлением данных, ее надо создать')
        return None
    else:
        connection = sqlite3.connect(db_name)
        del_old_lines(connection)
        if sw_src_file:
            update_switches_table(sw_src_file, connection)
        if dhcp_src_file:
            update_dhcp_table(dhcp_src_file, connection)

def del_old_lines(connection):
    now = datetime.today().replace(microsecond=0)
    week_ago = now - timedelta(days=7)
    connection.execute(f"DELETE FROM dhcp WHERE last_active < '{week_ago}'")

def update_switches_table(src_file, connection):
    with open(src_file) as f:
        data = yaml.safe_load(f)
    query = 'INSERT into switches values (?, ?)'
    for line in data['switches'].items():
        try:
            connection.execute(query, line)
        except sqlite3.IntegrityError as error:
            print(f'При добавлении данных: {line} Возникла ошибка: {error}')
    connection.commit()

def update_dhcp_table(src_file, connection):
    switch_name = re.match(r'\S*(?P<host>sw\d+)_dhcp_snooping.txt', src_file)
    src_data = open(src_file, 'r').read()
    connection.execute(f"UPDATE dhcp SET active = 0 WHERE switch = '{switch_name[1]}'")
    matches = re.findall(r'(?P<mac>\S+) +(?P<ip>\S+) +\d+ +\S+ +(?P<vlan>\d+) +(?P<intf>\S+)', src_data)

    query = "INSERT INTO dhcp VALUES (?, ?, ?, ?, ?, ?, datetime('now'))"
    for match in matches:
        list_to_append = list(match)
        list_to_append.extend([switch_name[1], 1])
        try:
            connection.execute(query, list_to_append)
        except sqlite3.IntegrityError:
            connection.execute(f"UPDATE dhcp SET ip = '{list_to_append[1]}',"
                               f" vlan = '{list_to_append[2]}',"
                               f" interface = '{list_to_append[3]}',"
                               f" active = '{list_to_append[5]}',"
                               f" last_active = datetime('now')"
                               f" WHERE mac = '{list_to_append[0]}' and switch = '{list_to_append[4]}'")

    connection.commit()

if __name__ == '__main__':
    update_db('dhcp_snooping.db', dhcp_src_file='new_data/sw2_dhcp_snooping.txt')


import AccountManager
import json
import xbmc
import os
import urllib

from cache import Cache, conditional_headers

''' Parameters for different F1TV API calls'''
__TV_API_PARAMS__ = {"event-occurrence": {"fields_to_expand": "image_urls,sessionoccurrence_urls,sessionoccurrence_urls__image_urls",
                               "fields": "name,self,image_urls,sessionoccurrence_urls,official_name,start_date,end_date,"
                                         "sessionoccurrence_urls__session_name,sessionoccurrence_urls__self,"
                                         "sessionoccurrence_urls__image_urls,sessionoccurrence_urls__start_time,sessionoccurrence_urls__available_for_user,sessionoccurrence_urls__content_urls"},
                     "race-season": {"fields": "year,name,self,eventoccurrence_urls,eventoccurrence_urls__name,eventoccurrence_urls__start_date,"
                                          "eventoccurrence_urls__self,eventoccurrence_urls__image_urls,image_urls",
                                "fields_to_expand": "eventoccurrence_urls,eventoccurrence_urls__image_urls,image_urls"},
                     "session-occurrence": {"fields": "name,self,image_urls,status,channel_urls,start_time,content_urls,session_name",
                                 "fields_to_expand": "channel_urls,image_urls,content_urls,channel_urls__image_urls,content_urls__image_urls,channel_urls__driver_urls,channel_urls__driver_urls__image_urls"},
                     "circuit": {"fields": "name,self,eventoccurrence_urls,eventoccurrence_urls__name,eventoccurrence_urls__start_date,"
                                          "eventoccurrence_urls__self,eventoccurrence_urls__image_urls,eventoccurrence_urls__official_name",
                                "fields_to_expand": "eventoccurrence_urls,eventoccurrence_urls__image_urls"}
                     }

class F1TV_API:
    """ Main API Object - is used to retrieve API information """

    def callAPI(self, endpoint, method="GET", api_ver=2, params=None, data=None):
        if int(api_ver) == 1:
            complete_url = 'https://f1tv.formula1.com' + endpoint
        elif int(api_ver) == 2:
            complete_url = 'https://f1tv-api.formula1.com/agl/1.0/gbr/en/all_devices/global/' + endpoint
        else:
            xbmc.log("Unable to make an API with invalid API version: {}".format(api_ver), xbmc.LOGERROR)
            return

        if method.upper() == 'GET':
            # Check to see if we've cached the response
            with Cache() as c:
                if params:
                    url_with_parameters = "{complete_url}?{parameters}".format(complete_url=complete_url,
                                                                               parameters=urllib.urlencode(params))
                else:
                    url_with_parameters = complete_url
                cached = c.get(url_with_parameters)
                if cached:
                    # If we have a fresh cached version, return it.
                    if cached["fresh"]:
                        return json.loads(cached["blob"])
                    # otherwise append applicable "If-None-Match"/"If-Modified-Since" headers
                    self.account_manager.getSession().headers.update(conditional_headers(cached))
                # request a new version of the data
                r = self.account_manager.getSession().get(complete_url, params=params, data=data)
                if 200 == r.status_code:
                    # add the new data and headers to the cache
                    c.set(url_with_parameters, r.content, r.headers)
                    return r.json()
                if 304 == r.status_code:
                    # the data hasn't been modified so just touch the cache with the new headers
                    # and return the existing data
                    c.touch(url_with_parameters, r.headers)
                    return json.loads(cached["blob"])

        elif method.upper() == 'POST':
            r = self.account_manager.getSession().post(complete_url, params=params, data=data)
            if r.ok:
                return r.json()
            else:
                return
        else:
            return

    def getFields(self, url):
        for key in __TV_API_PARAMS__:
            if key in url:
                return __TV_API_PARAMS__[key]

    def __init__(self):
        """ Initialize by creating AccountManager object"""
        self.account_manager = AccountManager.AccountManager()

    def login(self, username, password):
        """ Log in with supplied credentials."""
        return self.account_manager.login(username, password)

    def getStream(self, url):
        """ Get stream for supplied viewings item
            This will get the m3u8 url for Content and Channel."""
        item_dict = {"asset_url" if 'ass' in url else "channel_url": url}

        viewing_json = self.callAPI("/api/viewings/", api_ver=1, method='POST', data=json.dumps(item_dict))

        if 'chan' in url:
            return viewing_json["tokenised_url"]
        else:
            return viewing_json["objects"][0]["tata"]["tokenised_url"]

    def getSession(self, url):
        """ Get Session Object from API by supplying an url"""
        session = self.callAPI(url, params=__TV_API_PARAMS__["session-occurrence"])
        return session

    def getEvent(self, url, season = None):
        """ Get Event object from API by supplying an url"""
        event = self.callAPI(url, params=__TV_API_PARAMS__["event-occurrence"])
        return event

    def getSeason(self, url):
        """ Get Season object from API by supplying an url"""
        season = self.callAPI(url, api_ver=1, params=self.getFields(url))
        return season

    def getSeasons(self):
        """ Get all season urls that are available at API"""
        seasons = self.callAPI("/api/race-season/", api_ver=1, params={'order': '-year'})
        return seasons

    def getCircuits(self):
        """ Get all Circuit urls that are available at API"""
        circuits = self.callAPI("/api/circuit/", api_ver=1, params={"fields": "name,eventoccurrence_urls,self"})
        return circuits

    def getCircuit(self, url):
        """ Get Circuit object from API by supplying an url"""
        circuit = self.callAPI(url, api_ver=1, params=__TV_API_PARAMS__["circuit"])
        return circuit

    def getF2(self):
        f2 = self.callAPI("/api/sets/coll_4440e712d31d42fb95c9a2145ab4dac7")
        return f2

    def getSets(self):
        sets = self.callAPI("/api/sets/?slug=home", api_ver=1)
        content = {}
        for item in sets['objects'][0]['items']:
            item_details = self.callAPI(item['content_url'], api_ver=1)
            if 'title' in list(item_details):
                content[item_details['title']] = item['content_url']
            elif 'name' in list(item_details):
                content[item_details['name']] = item['content_url']
            else:
                content[item_details['UNKNOWN SET: ' + 'uid']] = item['content_url']
        return content

    def setLanguage(self, language):
        self.account_manager.session.headers['Accept-Language'] = "{}, en".format(language.upper())


import json
import os

from lEmoji.generator.web_extractor import WebExtractor
from lEmoji.data.tree import tree

class Generator:
    def __init__(self, proxies=None):
        self.extractor = WebExtractor(proxies=proxies)
        self.tree = self.extractor.build()
        self.working_dir = os.path.join(os.getcwd(), 'lEmoji')
        self.data_file = os.path.join(os.getcwd(), 'lEmoji', 'data', 'tree.py')

    @staticmethod
    def find_node(node_list, name):
        for node in node_list:
            if node['name'] == name:
                return node

    def generate(self, his_nodes=None, cur_nodes=None, version=None):
        if not version:
            his_nodes = tree
            cur_nodes = self.tree
        else:
            version = version.replace('.', '_')

        emoji_list = []

        for node in cur_nodes:
            his_node = self.find_node(his_nodes, node['name'])
            if node['type'] == 'folder':
                if not his_node:
                    self.generate([], node['children'], version=node['name'])
                elif node['time'] != his_node['time']:
                    self.generate(his_node['children'], node['children'], version=node['name'])
            elif node['name'].startswith('emoji'):
                emoji_list.extend(self.extractor.emoji_fetcher(node['link']))

        if version:
            path = os.path.join(self.working_dir, 'emoji_v%s.py' % version)
            emoji_list = list(set(emoji_list))
            with open(path, 'wb+') as f:
                s = 'EMOJI_LIST = ' + json.dumps(emoji_list, ensure_ascii=False)
                f.write(s.encode())
        else:
            with open(self.data_file, 'wb+') as f:
                s = 'tree = ' + json.dumps(cur_nodes, ensure_ascii=False) + '\n'
                f.write(s.encode())
            with open(os.path.join(self.working_dir, '__init__.py'), 'wb+') as f:
                for node in cur_nodes:
                    name = node['name'].replace('.', '_')
                    s = 'from .emoji_v%s import EMOJI_LIST as EMOJI_LIST_%s\n' % (
                        name, name.upper())
                    f.write(s.encode())

                s = '\n\nEMOJI_LIST = EMOJI_LIST_%s\n' % \
                    cur_nodes[-1]['name'].replace('.', '_').upper()
                f.write(s.encode())


import torch
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader
from torch import nn
from torch import optim
from lenet5 import Lenet5
from resNet import ResNet18
import time

def main():
    batchsz = 32

    cifar_train = datasets.CIFAR10('cifar', True, transform=transforms.Compose([
        transforms.Resize((32, 32)),
        transforms.ToTensor()
    ]), download=True)
    cifar_train = DataLoader(cifar_train, batch_size=batchsz, shuffle=True)

    cifar_test = datasets.CIFAR10('cifar', False, transform=transforms.Compose([
        transforms.Resize((32, 32)),
        transforms.ToTensor()
    ]), download=True)
    cifar_test = DataLoader(cifar_test, batch_size=batchsz, shuffle=True)
    x, label = iter(cifar_train).next()
    print('x:', x.shape, 'label:', label.shape)

    device = torch.device('cpu')
    # model = Lenet5().to(device)
    model = ResNet18().to(device)
    print(model)
    criteon = nn.CrossEntropyLoss().to(device)
    optimier = optim.Adam(model.parameters(), lr=1e-3)

    print(type(cifar_train))
    print((enumerate(cifar_train)))
    startTime = time.time()
    keepList = []
    for epoch in range(40):
        for batchidx, (x, label) in enumerate(cifar_train):
            x, label = x.to(device), label.to(device)
            logits = model(x)
            loss = criteon(logits, label)

            # backPropagation
            optimier.zero_grad()
            loss.backward()
            optimier.step()
        #
        print(epoch, loss.item())
        total_correct = 0
        total_num = 0
        model.eval()
        with torch.no_grad():
            for x, label in cifar_test:
                x, label = x.to(device), label.to(device)
                logits = model(x)
                pred = logits.argmax(dim=1)
                total_correct += torch.eq(pred, label).float().sum().item()
                total_num += x.size(0)
            acc = total_correct/total_num
            endTime = time.time()
            print(epoch, acc)
            keepList.append((endTime-startTime, acc))
            print('time:'+str(endTime - startTime))
            # startTime = time.time()
            if endTime - startTime > 20000:
                break
    file = open('resnet18.txt', 'w')
    for elem in keepList:
        cur = str(elem)
        file.write(cur+'\n')
    file.close()

if __name__ == '__main__':
    main()


from django.contrib import admin
from django.urls import path, include
from catalog import views

urlpatterns = [
    path('', views.catalog, name='catalog'),
    path('category/<slug>', views.category, name='category'),
    path('products', views.products, name='products'),
    path('products/<slug>', views.product, name='product'),
    path('add_product', views.add_product, name='add_product'),
    path('cart', views.cart, name='cart'),
    path('make_order', views.make_order, name='make_order'),
    path('api/get_attributes', views.GetAttributesView.as_view()),
    path('api/get_sel_attributes', views.GetSelectableAttributesView.as_view()),
]


# nrbf.py - .NET Remoting Binary Format reading library for Python 3.6+
# Copyright (C) 2017 Christopher Gurnee
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

__all__ = ['read_stream', 'serialization', 'JSONEncoder']

import enum, itertools, json, re, sys
from array       import array as Array, typecodes
from collections import OrderedDict, namedtuple
from copy        import deepcopy
from datetime    import datetime, timedelta, timezone
from decimal     import Decimal
from keyword     import iskeyword
from struct      import Struct, calcsize, pack
from namedlist   import namedlist

# Decorator which adds an enum value (an int or a length-one bytes) and its associated
# reader function to a registration dict (either PrimitiveType_readers or RecordType_readers)
def _register_reader(type_dict, enum_value):
    if isinstance(enum_value, int):
        enum_value = bytes([enum_value])  # convert from int to length-one bytes
    else:
        assert isinstance(enum_value, bytes) and len(enum_value) == 1
    def decorator(read_func):
        type_dict[enum_value] = read_func
        return read_func
    return decorator

class serialization:
    def __init__(self, streamfile, can_overwrite_member = False):
        '''
        :param streamfile: a file-like object in .NET Remoting Binary Format
        :param can_overwrite_member: iff True, overwrite_member() may be called
        '''
        assert streamfile.readable()
        if can_overwrite_member:
            assert streamfile.writable()
            assert streamfile.seekable()
        self._file                  = streamfile
        self._Class_by_id           = {}    # see _read_ClassInfo()
        self._objects_by_id         = {}    # all referenceable objects indexed by ObjectId
        self._root_id               = None  # the id of the single root object
        self._member_references     = []    # all seen simple references, see _read_MemberReference()
        self._collection_references = []    # all seen collection references, e.g. .NET dicts and lists
        self._add_overwrite_info     = can_overwrite_member
        self._overwrite_info_by_pyid = {}   # if above is True, overwrite_info objs indexed by python id()

    # Below are a set of "readers" and other support types, one per defined structure
    # in revision 11.0 of the ".NET Remoting: Binary Format Data Structure" specification
    # in approximately the same order of each structure's definition. Each one reads in
    # the structure's elements, and depending on the reader type may return some or all
    # of the read values. This first section implements the Primitive Types, each one
    # returning the closest Python analogous type to the Primitive Type read.
    ######## Reference: https://msdn.microsoft.com/en-us/library/cc236844.aspx ########

    # A dict of all PrimitiveType readers indexed by a length-one PrimitiveTypeEnumeration bytes object
    _PrimitiveType_readers = {}

    @_register_reader(_PrimitiveType_readers, 3)
    def _read_Char(self):
        utf8_bytes = bytearray()
        while True:
            utf8_bytes += self._file.read(1)
            try:
                return utf8_bytes.decode('utf-8')
            except UnicodeDecodeError:
                if len(utf8_bytes) > 4:
                    raise

    @_register_reader(_PrimitiveType_readers, 12)
    def _read_TimeSpan(self):
        return timedelta(microseconds= self._read_Int64() / 10)  # units of 100 nanoseconds

    @_register_reader(_PrimitiveType_readers, 13)
    def _read_DateTime(self):
        ticks = self._read_UInt64()
        kind  = ticks >> 62     # the 2 most significant bits store the "kind"
        ticks &= (1 << 62) - 1  # all but the above
        if ticks >= 1 << 61:    # if negative, reinterpret
            ticks -= 1 << 62    # as 62-bit two's complement
        time = datetime(1, 1, 1)
        try:
            time += timedelta(microseconds= ticks / 10)
        except OverflowError:
            pass
        if kind == 1:
            time = time.replace(tzinfo=timezone.utc)
        elif kind == 2:
            try:
                time = time.astimezone()  # kind 2 is the local time zone
            except OSError:
                pass
        return time

    @_register_reader(_PrimitiveType_readers, 18)
    def _read_LengthPrefixedString(self):
        length = 0
        for bit_range in range(0, 5*7, 7):  # each bit range is 7 bits long, with at most 5 of them
            byte    = self._read_Byte()
            length += (byte & ((1 << 7) - 1)) << bit_range  # highest bit masked out, then shifted
            if byte & (1 << 7) == 0:        # the "is there more?" (highest/most significant) bit
                break
        else:
            raise OverflowError('LengthPrefixedString overflow')
        return self._file.read(length).decode('utf-8')

    @_register_reader(_PrimitiveType_readers, 5)
    def _read_Decimal(self):
        return Decimal(self._read_LengthPrefixedString())

    @_register_reader(_PrimitiveType_readers, 17)
    def _read_Null(self):
        return None

    # Creates the rest of the PrimitiveType readers (those based on struct.unpack);
    # this only works when it's called after the class has been fully defined.
    _array_format_by_primitive_type  = {}  # array-format-specs  indexed by PrimitiveTypeEnumeration length-one bytes
    _struct_format_by_primitive_type = {}  # struct-format-specs indexed by PrimitiveTypeEnumeration length-one bytes
    @classmethod
    def _create_PrimitiveType_readers(cls):
        for enum_value, name, format in (
                ( 1, '_read_Boolean',  '?'),
                ( 2, '_read_Byte',     'B'),
                ( 6, '_read_Double',  '<d'),
                ( 7, '_read_Int16',   '<h'),
                ( 8, '_read_Int32',   '<l'),
                ( 9, '_read_Int64',   '<q'),
                (10, '_read_SByte',    'b'),
                (11, '_read_Single',  '<f'),
                (14, '_read_UInt16',  '<H'),
                (15, '_read_UInt32',  '<L'),
                (16, '_read_UInt64',  '<Q')):
            enum_value = bytes([enum_value])  # convert from int to length-one bytes
            if format[-1] in typecodes:  # if the format can also be used to create an array type
                cls._array_format_by_primitive_type[enum_value] = format[-1]  # (the [-1] skips the '<' flag)
            cls._struct_format_by_primitive_type[enum_value] = format
            struct = Struct(format)
            length = calcsize(format)
            reader = lambda self, s=struct, l=length: s.unpack(self._file.read(l))[0]
            setattr(cls, name, _register_reader(cls._PrimitiveType_readers, enum_value)(reader))

    # The ClassTypeInfo structure is read and ignored
    def _read_ClassTypeInfo(self):
        self._read_LengthPrefixedString()  # TypeName
        self._read_Int32()                 # LibraryId

    @enum.unique
    class _MessageFlags(enum.Flag):
        NoArgs                 = 0x00000001
        ArgsInline             = 0x00000002
        ArgsIsArray            = 0x00000004
        ArgsInArray            = 0x00000008
        NoContext              = 0x00000010
        ContextInline          = 0x00000020
        ContextInArray         = 0x00000040
        MethodSignatureInArray = 0x00000080
        PropertiesInArray      = 0x00000100
        NoReturnValue          = 0x00000200
        ReturnValueVoid        = 0x00000400
        ReturnValueInline      = 0x00000800
        ReturnValueInArray     = 0x00001000
        ExceptionInArray       = 0x00002000
        GenericMethod          = 0x00008000

    def _read_ValueWithCode(self):
        return self._PrimitiveType_readers[self._file.read(1)](self)

    _read_StringValueWithCode = _read_ValueWithCode

    def _read_ArrayOfValueWithCode(self):
        return [self._read_ValueWithCode() for i in range(self._read_Int32())]

    # A dict of all RecordType readers indexed by a length-one RecordTypeEnumeration bytes object
    _RecordType_readers = {}

    @_register_reader(_RecordType_readers, 21)
    def _read_BinaryMethodCall(self):
        raise NotImplementedError('BinaryMethodCall')
        # flags = self._MessageFlags(self._read_Int32())  # MessageEnum
        # self._read_StringValueWithCode()                # MethodName
        # self._read_StringValueWithCode()                # TypeName
        # if flags & self._MessageFlags.ContextInline:
        #     self._read_StringValueWithCode()            # CallContext
        # if flags & self._MessageFlags.ArgsInline:
        #     self._read_ArrayOfValueWithCode()           # Args

    @_register_reader(_RecordType_readers, 22)
    def _read_BinaryMethodReturn(self):
        raise NotImplementedError('BinaryMethodReturn')
        # flags = self._MessageFlags(self._read_Int32())  # MessageEnum
        # if flags & self._MessageFlags.ReturnValueInline:
        #     self._read_ValueWithCode()                  # ReturnValue
        # if flags & self._MessageFlags.ContextInline:
        #     self._read_StringValueWithCode()            # CallContext
        # if flags & self._MessageFlags.ArgsInline:
        #     self._read_ArrayOfValueWithCode()           # Args

    ######## Classes ########

    # Reads a ClassInfo structure, and if the Class's ObjectId hasn't yet been seen, adds to
    # self._Class_by_id a new Python class with the same members as the ClassInfo specifies.
    def _read_ClassInfo(self):
        object_id    = self._read_Int32()
        class_name   = self._read_LengthPrefixedString()
        member_count = self._read_Int32()
        member_names = [self._read_LengthPrefixedString() for i in range(member_count)]
        unique_members = set()
        for member_num, member_name in enumerate(member_names):
            member_name = make_unique(sanitize_identifier(member_name), unique_members)
            unique_members.add(member_name)
            member_names[member_num] = member_name
        Class = namedlist(sanitize_identifier(class_name), member_names, default=None)
        Class._id = object_id
        Class._primitive_types = {}  # filled in below by _read_MemberTypeInfo()
        Class._is_system_class = False
        self._Class_by_id[object_id] = Class
        return Class

    # Readers for the AdditionalInfos member of MemberTypeInfo indexed by
    # BinaryTypeEnumeration ints; created after this class is fully defined
    _AdditionalInfo_readers = ()

    def _read_MemberTypeInfo(self, Class):
        binary_types = [self._read_Byte() for m in Class._fields]  # BinaryTypeEnums
        for member_num, binary_type in enumerate(binary_types):    # AdditionalInfos
            additional_info = self._AdditionalInfo_readers[binary_type](self)
            if binary_type == 0:  # (0 == BinaryTypeEnumeration.Primitive)
                Class._primitive_types[member_num] = additional_info  # save any PrimitiveType for later

    @_register_reader(_RecordType_readers, 5)
    def _read_ClassWithMembersAndTypes(self):
        Class = self._read_ClassInfo()
        self._read_MemberTypeInfo(Class)
        self._read_Int32()  # LibraryId is ignored
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 3)
    def _read_ClassWithMembers(self):
        Class = self._read_ClassInfo()
        self._read_Int32()  # LibraryId is ignored
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 4)
    def _read_SystemClassWithMembersAndTypes(self):
        Class = self._read_ClassInfo()
        Class._is_system_class = True
        self._read_MemberTypeInfo(Class)
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 2)
    def _read_SystemClassWithMembers(self):
        Class = self._read_ClassInfo()
        Class._is_system_class = True
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 1)
    def _read_ClassWithId(self):
        object_id = self._read_Int32()
        Class = self._Class_by_id[self._read_Int32()]  # MetadataId
        return self._read_members_into(Class(), object_id, Class._primitive_types.get)

    # If primitive_type is not None, read the specified primitive_type with one of the
    # PrimitiveType_readers. Otherwise read the next RecordType in the streamfile with
    # one of the RecordType_readers. If overwrite_info is not None, add overwrite info
    # (a tuple containing: file position, struct type) to overwrite_info[overwrite_index]
    # if the value read is an overwritable primitive. Finally, returns the value read.
    def _read_Record_or_Primitive(self, primitive_type, overwrite_info = None, overwrite_index = None):
        if primitive_type:
            if overwrite_info is not None:
                format = self._struct_format_by_primitive_type.get(primitive_type)
                if format:
                    overwrite_info[overwrite_index] = self._file.tell(), format
            return self._PrimitiveType_readers[primitive_type](self)
        else:
            record_type = self._file.read(1)
            # If overwrite_info is not None and record_type == MemberPrimitiveTyped, parse it ourselves--
            # read in the PrimitiveTypeEnum and call ourselves to finish parsing and add the overwrite_info
            if overwrite_info is not None and record_type == b'\x08':
                return self._read_Record_or_Primitive(self._file.read(1), overwrite_info, overwrite_index)
            return self._RecordType_readers[record_type](self)

    _Reference = namedlist('_Reference',  # see _read_MemberReference()
        'id parent index_in_parent resolved collection_resolver orig_obj', default=None)

    # Reads members or array elements into the 'obj' pre-allocated list or class instance
    def _read_members_into(self, obj, object_id, members_primitive_type):
        assert callable(members_primitive_type)  # when called with the member_num, returns any respective primitive type
        if self._add_overwrite_info:
            # create the object which will store the overwrite info for all the members in obj
            overwrite_info = [None] * len(obj) if isinstance(obj, list) else obj.__class__()
        else:
            overwrite_info = None
        member_num = 0
        while member_num < len(obj):
            primitive_type = members_primitive_type(member_num)
            val = self._read_Record_or_Primitive(primitive_type, overwrite_info, member_num)
            if isinstance(val, self._BinaryLibrary):  # a BinaryLibrary can precede the actual member, it's ignored
                val = self._read_Record_or_Primitive(primitive_type, overwrite_info, member_num)
            if isinstance(val, self._ObjectNullMultiple):  # represents one or more empty members
                member_num += val.count
                continue
            if isinstance(val, self._Reference):      # see _read_MemberReference()
                val.parent          = obj
                val.index_in_parent = member_num
            obj[member_num] = val
            member_num += 1
        if overwrite_info is not None:
            self._overwrite_info_by_pyid[id(obj)] = overwrite_info
        # If this object is a .NET collection (e.g. a Generic dict or list) which can be
        # replaced by a native Python type, insert a collection _Reference instead of the raw
        # object which will be resolved later in read_stream() using a "collection resolver"
        if getattr(obj.__class__, '_is_system_class', False):
            for collection_name, resolver in self._collection_resolvers:
                if obj.__class__.__name__.startswith(f'System_Collections_Generic_{collection_name}_'):
                    obj = self._Reference(object_id, collection_resolver=resolver, orig_obj=obj)
                    self._collection_references.append(obj)
                    break
        self._objects_by_id[object_id] = obj
        return obj

    ######## Arrays ########

    @enum.unique
    class _BinaryArrayTypeEnumeration(enum.Enum):
        Single            = 0
        Jagged            = 1
        Rectangular       = 2
        SingleOffset      = 3
        JaggedOffset      = 4
        RectangularOffset = 5
        def is_offset(self):
            return self in (self.SingleOffset, self.JaggedOffset, self.RectangularOffset)
        def is_rectangular(self):
            return self in (self.Rectangular, self.RectangularOffset)

    def _read_ArrayInfo(self):
        return (self._read_Int32(),  # ObjectId
                self._read_Int32())  # Length

    # This might not be able to use _read_members_into() since BinaryArrays are much more complex than
    # other types, thus it has no choice but to reimplement much of what _read_members_into() does
    @_register_reader(_RecordType_readers, 7)
    def _read_BinaryArray(self):
        object_id  = self._read_Int32()
        array_type = self._BinaryArrayTypeEnumeration(self._read_Byte())
        rank       = self._read_Int32()
        lengths    = [self._read_Int32() for i in range(rank)]
        if array_type.is_offset():
            [self._read_Int32() for i in range(rank)]  # LowerBounds are not implemented; they're ignored
        binary_type     = self._read_Byte()
        additional_info = self._AdditionalInfo_readers[binary_type](self)
        primitive_type  = additional_info if binary_type == 0 else None  # (0 == BinaryTypeEnumeration.Primitive)
        # If the BinaryArray is multidimensional, the complex code branch is required:
        if array_type.is_rectangular():
            array_format = self._array_format_by_primitive_type.get(primitive_type) if primitive_type else None
            if array_format:  # if not None, a Python Array can be used for the *last* dimension...
                array_length = lengths.pop()  # ...which is removed from the lengths list here
            array = multidimensional_array(lengths)  # preallocate a list of lists (it's not a Python Array)
            skip  = 0
            for indexes in itertools.product(*[range(l) for l in lengths]):  # iterates through all of the indexes
                if array_format:
                    val = self._read_Array_native_elements(array_format, array_length)  # read in the last index in one call
                else:
                    if skip > 0:
                        skip -= 1
                        continue
                    val = self._read_Record_or_Primitive(primitive_type)
                    if isinstance(val, self._BinaryLibrary):   # a BinaryLibrary can precede the actual array element, it's ignored
                        val = self._read_Record_or_Primitive(primitive_type)
                    if isinstance(val, self._ObjectNullMultiple):  # represents one or more empty elements
                        skip = val.count - 1  # counts this iteration which we're skipping right now
                        continue
                    if isinstance(val, self._Reference):       # see _read_MemberReference()
                        list_indexes = ''.join(f'[{i}]' for i in indexes[:-1])
                        val.parent          = eval(f'array{list_indexes}')  # the parent list, i.e. all but the last index
                        val.index_in_parent = indexes[-1]                   # the last index
                list_indexes = ''.join(f'[{i}]' for i in indexes)
                exec(f'array{list_indexes} = val')
            self._objects_by_id[object_id] = array
            return array
        else:  # else it's not multidimensional, the standard code branch can be used:
            assert len(lengths) == 1
            return self._read_Array_elements(lengths[0], object_id, primitive_type)

    @_register_reader(_RecordType_readers, 16)
    def _read_ArraySingleObject(self):
        object_id, length = self._read_ArrayInfo()
        return self._read_Array_elements(length, object_id)

    @_register_reader(_RecordType_readers, 15)
    def _read_ArraySinglePrimitive(self):
        object_id, length = self._read_ArrayInfo()
        primitive_type    = self._file.read(1)
        return self._read_Array_elements(length, object_id, primitive_type)

    _read_ArraySingleString = _register_reader(_RecordType_readers, 17)(_read_ArraySingleObject)

    def _read_Array_elements(self, length, object_id, primitive_type = None):
        array_format = self._array_format_by_primitive_type.get(primitive_type) if primitive_type else None
        if array_format:  # if not None, a Python Array can be used
            array = self._read_Array_native_elements(array_format, length)
            self._objects_by_id[object_id] = array
            return array
        return self._read_members_into([None] * length, object_id, lambda i: primitive_type)

    assert sys.byteorder in ('little', 'big')
    def _read_Array_native_elements(self, format, length):
        if self._add_overwrite_info:
            pos = self._file.tell()
        array = Array(format)
        array.fromfile(self._file, length)  # read them in one call
        if sys.byteorder == 'big':
            array.byteswap()
        if self._add_overwrite_info:
            format = '<' + format  # always little-endian
            self._overwrite_info_by_pyid[id(array)] = [
                (pos + offset, format) for offset in range(0, length * array.itemsize, array.itemsize) ]
        return array

    _read_MemberPrimitiveTyped = _register_reader(_RecordType_readers, 8)(_read_ValueWithCode)

    # Wherever an ObjectId is encountered above, it's added to the self._objects_by_id map
    # along with the newly-created object. A MemberReference contains an IdRef which refers
    # to an object in this map, however because MemberReferences can appear before the object
    # to which they refer, they can't be resolved until the very end. Instead, a _Reference
    # Python object is temporarily created which stores enough information (its parent and
    # its index inside the parent) to eventually replace it with the actual object referenced
    # once all of the objects have been read from the stream in read_stream().
    @_register_reader(_RecordType_readers, 9)
    def _read_MemberReference(self):
        member_ref = self._Reference(self._read_Int32())  # IdRef
        self._member_references.append(member_ref)  # (parent and index_in_parent should be set by the caller)
        return member_ref

    _read_ObjectNull = _register_reader(_RecordType_readers, 10)(_read_Null)

    _ObjectNullMultiple = namedtuple('_ObjectNullMultiple', 'count')

    @_register_reader(_RecordType_readers, 14)
    def _read_ObjectNullMultiple(self):
        return self._ObjectNullMultiple(self._read_Int32())

    @_register_reader(_RecordType_readers, 13)
    def _read_ObjectNullMultiple256(self):
        return self._ObjectNullMultiple(self._read_Byte())

    @_register_reader(_RecordType_readers, 6)
    def _read_BinaryObjectString(self):
        object_id = self._read_Int32()
        string    = self._read_LengthPrefixedString()
        self._objects_by_id[object_id] = string
        return string

    @_register_reader(_RecordType_readers, 0)
    def _read_SerializationHeaderRecord(self):
        self._root_id = self._read_Int32()
        self._read_Int32()  # HeaderId is ignored
        major_version = self._read_Int32()
        minor_version = self._read_Int32()
        if major_version != 1:
            raise NotImplementedError(f'SerializationHeaderRecord.MajorVersion == {major_version}')
        if minor_version != 0:
            raise NotImplementedError(f'SerializationHeaderRecord.MinorVersion == {minor_version}')
        if self._root_id == 0:
            raise NotImplementedError('SerializationHeaderRecord.RootId == 0')

    class _BinaryLibrary: pass
    @_register_reader(_RecordType_readers, 12)
    def _read_BinaryLibrary(self):
        self._read_Int32()                 # MinorVersion and
        self._read_LengthPrefixedString()  # LibraryName are ignored
        return self._BinaryLibrary()

    class _MessageEnd: pass
    @_register_reader(_RecordType_readers, 11)
    def _read_MessageEnd(self):
        return self._MessageEnd()

    def read_header(self):
        '''Reads just the SerializationHeaderRecord from the streamfile.
        It's not necessary to call this, however it may be called before read_stream() if desired.
        Otherwise, read_stream() will raise an exception if the SerializationHeaderRecord isn't found.

        :return: True if the streamfile is in a supported .NET Remoting Binary Format
        '''
        assert self._root_id is None, 'read_header() has not already been called'
        try:
            self._read_Record_or_Primitive(primitive_type=False)
        except Exception:
            return False
        return self._root_id is not None

    def read_stream(self):
        '''Read the streamfile in .NET Remoting Binary Format and extract its root object

        :return: the root object contained in the stream
        '''
        if self._root_id is None and not self.read_header():
            raise RuntimeError('SerializationHeaderRecord not found (probably not an NRBF file)')
        obj = None
        while not isinstance(obj, self._MessageEnd):
            obj = self._read_Record_or_Primitive(primitive_type=False)
        self._Class_by_id.clear()

        # Resolve all the collection references
        for reference in self._collection_references:
            replacement = reference.collection_resolver(self, reference)  # calls one of the non-simple resolvers below
            # The final steps common to all collection resolvers are completed below
            if reference.parent:
                reference.parent[reference.index_in_parent] = replacement
            self._objects_by_id[reference.id] = replacement
        self._collection_references.clear()

        # Resolve all the (remaining) simple member references
        for reference in self._member_references:
            self._resolve_simple_reference(reference)
        self._member_references.clear()

        obj = self._objects_by_id[self._root_id]
        self._objects_by_id.clear()
        self._root_id = None
        return obj

    # Convert a _Reference representing a .NET dictionary collection into a Python dict
    def _resolve_dict_reference(self, dict_ref):
        orig_obj = dict_ref.orig_obj
        # If KeyValuePairs is itself a _Reference, it must be resolved first
        if isinstance(orig_obj.KeyValuePairs, self._Reference):
            self._resolve_simple_reference(orig_obj.KeyValuePairs)
        replacement    = {}
        overwrite_info = {} if self._add_overwrite_info else None
        for item in orig_obj.KeyValuePairs:
            try:
                # If any key is a _Reference, it must be resolved first
                # (value _References will be resolved later)
                if isinstance(item.key, self._Reference):
                    self._resolve_simple_reference(item.key)
                assert item.key not in replacement
                replacement[item.key] = item.value
                if overwrite_info is not None:
                    overwrite_info[item.key] = self._overwrite_info_by_pyid[id(item)].value
            except (AssertionError, TypeError):  # not all .NET dictionaries can be converted to Python dicts;
                replacement = orig_obj           # if the conversion fails, just proceed w/the original object
                break
        else:
            # If any dict value is a _Reference, fix its parent and index_in_parent
            for key, value in replacement.items():
                if isinstance(value, self._Reference):
                    value.parent          = replacement
                    value.index_in_parent = key
            if overwrite_info is not None:
                self._overwrite_info_by_pyid[id(replacement)] = overwrite_info
        return replacement

    # Convert a _Reference representing a .NET list collection into a Python list
    def _resolve_list_reference(self, list_ref):
        orig_obj = list_ref.orig_obj
        # If items is itself a _Reference, it must be resolved first
        if isinstance(orig_obj.items, self._Reference):
            self._resolve_simple_reference(orig_obj.items)
        replacement = orig_obj.items
        # If any list element is a _Reference, fix its parent
        for element in replacement:
            if isinstance(element, self._Reference):
                element.parent = replacement  # (the index_in_parent remains the same)
        return replacement

    _collection_resolvers = (
        ('Dictionary', _resolve_dict_reference),
        ('List',       _resolve_list_reference)
    )

    # Convert a _Reference representing a MemberReference into its referenced object
    def _resolve_simple_reference(self, member_ref):
        assert not member_ref.collection_resolver
        if member_ref.resolved:
            return
        replacement = self._objects_by_id[member_ref.id]
        member_ref.parent[member_ref.index_in_parent] = replacement
        member_ref.resolved = True

    def overwrite_member(self, obj, member, value):
        '''Overwrite an object's member with a new value in the streamfile (does not change obj).

        :param obj: an object in the hierarchy returned by read_stream()
        :param member: a writable member (attribute) name, index (for lists/arrays), or key (for dicts) in obj
        :param value: the new value to be written to the streamfile
        '''
        assert self._add_overwrite_info, 'serialization object must have been constructed with can_overwrite_member == True'
        overwrite_info = self._overwrite_info_by_pyid[id(obj)]
        if isinstance(member, int) or isinstance(obj, dict):
            pos, format = overwrite_info[member]
        else:
            pos, format = getattr(overwrite_info, member)
        value   = pack(format, value)
        old_pos = self._file.tell()
        try:
            self._file.seek(pos)
            self._file.write(value)
        finally:
            self._file.seek(old_pos)

    def is_member_writable(self, obj, member):
        '''Returns True if the object's member can be overwritten.
        Can (but isn't guaranteed to) raise an exception if the member doesn't exist.

        :param obj: an object in the hierarchy returned by read_stream()
        :param member: a member (attribute) name, index (for lists/arrays), or key (for dicts) in obj
        '''
        assert self._add_overwrite_info, 'serialization object must have been constructed with can_overwrite_member == True'
        overwrite_info = self._overwrite_info_by_pyid.get(id(obj))
        if overwrite_info is None:
            return False
        if isinstance(member, int) or isinstance(obj, dict):
            return overwrite_info[member]          is not None
        else:
            return getattr(overwrite_info, member) is not None

# Finish setting up the serialization class
serialization._create_PrimitiveType_readers()
#
serialization._AdditionalInfo_readers = (
    lambda self: self._file.read(1),           # 0 Primitive
    serialization._read_Null,                  # 1 String
    serialization._read_Null,                  # 2 Object
    serialization._read_LengthPrefixedString,  # 3 SystemClass
    serialization._read_ClassTypeInfo,         # 4 Class
    serialization._read_Null,                  # 5 ObjectArray
    serialization._read_Null,                  # 6 StringArray
    lambda self: self._file.read(1),           # 7 PrimitiveArray
)

# Now that we're done adding PrimitiveType and RecordType readers, ensure they're
# all present (except PrimitiveType 4 and RecordTypes 18-20 which aren't defined)
assert all(bytes([i]) in serialization._PrimitiveType_readers if i != 4            else True for i in range(1, 19))
assert all(bytes([i]) in serialization._RecordType_readers    if not 18 <= i <= 20 else True for i in range(23))

def read_stream(streamfile):
    '''Read a file in .NET Remoting Binary Format and extract its root object

    :param streamfile: a file-like object
    :return: the root object contained in the stream
    '''
    return serialization(streamfile).read_stream()

# A JSONEncoder which can convert an object returned by read_stream() into json
# (can't handle circular references; primarily intended for debugging purposes)
class JSONEncoder(json.JSONEncoder):
    def default(self, o):
        if hasattr(o, '_asdict'):
            d = OrderedDict(_class_name=o.__class__.__name__)  # prepend the class name
            d.update(o._asdict())
            return d
        if isinstance(o, Array):
            return o.tolist()
        if isinstance(o, (datetime, timedelta)):
            return str(o)
        if isinstance(o, Decimal):
            return repr(o)
        return super().default(o)

# Returns a version of the identifier suitable to pass to namedlist
_identifier_re = re.compile('[^a-z0-9_]', flags=re.IGNORECASE)
def sanitize_identifier(identifier):
    identifier = _identifier_re.sub('_', identifier).lstrip('0123456789_')
    if not identifier:
        return 'invalid_identifier'
    if iskeyword(identifier):
        identifier += '_'
    assert identifier.isidentifier()
    return identifier

# Returns a version of the name which isn't present in the unique_set
def make_unique(name, unique_set):
    if name not in unique_set:
        return name
    for append in itertools.count(2):
        replacement = name + str(append)
        if replacement not in unique_set:
            return replacement

# Pre-allocates a "multidimensional array", i.e. a list of lists of Nones
def multidimensional_array(lengths):
    if not lengths:
        return None
    arrays = [None] * lengths[-1]
    for length in reversed(lengths[:-1]):
        arrays = [deepcopy(arrays) for i in range(length)]
    return arrays

if __name__ == '__main__':
    if len(sys.argv) != 2:
        sys.exit(f'Usage: {sys.argv[0]} streamfile')
    json_encoder = JSONEncoder(indent=4)
    with open(sys.argv[1], 'rb') as streamfile:
        while True:
            print(json_encoder.encode(read_stream(streamfile)))
            if streamfile.peek(1) == b'':
                break


from urllib.request import urlopen
from urllib.error import HTTPError
import json
from colorama import Fore, Style, Back

cmds = {"international": ["BBC News", "CNN", "Daily Mail", "The Guardian UK", "The New York Times", "The Telegraph"],
        "national": ["The Times of India", "The Hindu"],
        "technology": ["Engadget", "Hacker News", "Techcrunch", "TechRadar", "The Next Web"],
        "sports": ["BBC Sport", "ESPN", "ESPN Cric info", "Fox Sports", "TalkSport"],
        "finance": ["Bloomberg", "Business Insider", "Financial Times", "The Wall Street Journal", "Time", "Fortune"],
        "entertainment": ["BuzzFeed", "Entertainment Weekly", "MTV News", "MTV News UK"],
        "science": ["New Scientist"],
        "blog": ["Reddit r all"],
        "adventure": ["National Geographic"],
        "help": ["help", "exit","list-news","sources"],
        "list-news": ["international", "national", "technology", "sports", "finance", "entertainment", "science", "blog", "adventure"]
        }
api_key = None

def news(source):
    print(" 1. Top News\n 2. Latest News\n 3. Most Popular News\n 4. Return")
    loop = 1
    while loop:
        flag = int(input("[1/2/3/4] >_ "))
        if flag == 1:
            url = "https://newsapi.org/v1/articles?source=" + source + "&sortBy=top&apiKey=" + api_key
            loop = 0
        elif flag == 2:
            url = "https://newsapi.org/v1/articles?source=" + source + "&sortBy=latest&apiKey=" + api_key
            loop = 0
        elif flag == 3:
            url = "https://newsapi.org/v1/articles?source=" + source + "&sortBy=popular&apiKey=" + api_key
            loop = 0
        elif flag == 4:
            loop = 0
            return console()
        else:
            print("Enter a valid no.")
    try:
        with urlopen(url) as httpob:
            decob = httpob.read().decode("utf-8")
            jsonob = json.loads(decob)
            news = jsonob["articles"]
    except HTTPError as e:
        print("Invalid API")
        create_api_file(file_name)
        return console()
    draw_border()
    for n in news:
        print(Fore.RESET)
        try:
            print(Back.YELLOW + src2(jsonob["source"]) + Back.RESET)
            print((Style.BRIGHT +"By: " + n["author"]) + Style.RESET_ALL)
        except:
        	print((Style.BRIGHT +"By: " + src2(jsonob["source"])) + Style.RESET_ALL)
        try:
            print(Back.RED + (Style.BRIGHT + n["title"] + Style.RESET_ALL))
            print(Fore.BLUE + n["description"] + Fore.RESET)
        except:
            print(Fore.RED + "SOME ERROR OCCURED\n" + Fore.RESET)
        print(Back.BLUE +(Style.BRIGHT + "url: "+ n["url"]) + Style.RESET_ALL + Back.RESET)
        try:
        	print(Fore.GREEN + "Published At: "+ n["publishedAt"] + Fore.RESET )
        except:
        	draw_border()
        	continue
        draw_border()

def sour():
    url = "https://newsapi.org/v1/sources?language=en"
    try:
        with urlopen(url) as httpob:
            decob = httpob.read().decode("utf-8")
            jsonob = json.loads(decob)
            sources = jsonob["sources"]

    except HTTPError as e:
        print("Invalid API")
        create_api_file(file_name)
        return console()

    draw_border()
    key = 1
    for s in sources:
        print(Fore.RESET)
        try:
            print(key, end="")
            print(". " + Back.BLUE + (Style.BRIGHT +s["name"] + Style.RESET_ALL) + Back.RESET)
            key = key+1
        except:
            print(Fore.RED + "SOME ERROR OCCURED\n" + Fore.RESET)
    draw_border()
    print("Enter the index of any source if you want to know more about it.\nEnter -1 for returning back to main menu\n")
    loop = True
    while loop:
        flag = input(">")
        try:
            if flag == '-1':
                loop = False
            elif int(flag)>0 or int(flag)<60:
                s = sources[int(flag) -1]
                print('\n')
                print(Back.BLUE + (Style.BRIGHT +s["name"] + Style.RESET_ALL) + Back.RESET)
                print("Description: " + Fore.BLUE + s["description"] + Fore.RESET)
                print("Category: " + Fore.RED + s["category"] + Fore.RESET)
                print("Url: " + Fore.GREEN + (Style.BRIGHT + s["url"]) + Fore.RESET)
                print("\nEnter any other index. Enter -1 to exit\n")
        except:
            print("Invalid Entry\n")

def draw_border():
    width = 80
    print(Style.BRIGHT+"-" * width +Style.RESET_ALL+"\n")

def src(n):
    k = n.replace(" ", "-")
    return k

def src2(n):
	k = n.replace("-"," ")
	return k

def create_api_file(file_name):
    global api_key
    api_key = input("Enter a valid API key: ")

    with open(file_name, "w") as f:
        f.write(api_key + '\n')
    f.close()
    print("The API key is: " + api_key)

def get_api():
    global api_key
    global file_name
    file_name = "user-api.txt"

    try:
        f = open(file_name, "r")
        api_key = f.readline()
        f.close()
        print("The API key is: " + api_key)
        print("Do you want to change the API key? [Y/N]: ")
        flag = input()
        if flag == 'Y':
            create_api_file(file_name)
    except FileNotFoundError:
        create_api_file(file_name)
    if api_key == None or len(api_key) == 0:
        create_api_file(file_name)

def console():
    while True:
        cmd = input("(topic) >_ ")
        if (cmd not in cmds["list-news"] and cmd not in cmds["help"]):
            print(Fore.RED + "WRONG COMMAND")
            print(Fore.GREEN + "Try these COMMANDS" + Fore.RESET)
            for c in cmds["help"]:
                print("    " + c)
        elif cmd == "help":
            print(Fore.GREEN + "Try these COMMANDS" + Fore.RESET)
            for c in cmds["help"]:
                print("    " + c)
        elif cmd == "list-news":
            print(Fore.GREEN + "Find news about any of these topics" + Fore.RESET)
            for l in cmds["list-news"]:
                print("    " + l)
        elif cmd == "exit":
            print(Style.RESET_ALL)
            exit()
        elif cmd == "sources":
            sour()
        else:
            for n in cmds[cmd]:
                s = src(n)
                news(s)

if __name__ == "__main__":
    print("\nPowered by NewsAPI.org\n")
    get_api()
    console()


import gym
import numpy as np
import torch
from modules import AdversarialDQN

import pdb
def eval_policy(policy, env_name, eval_episodes=10, seed = 0):
    env_seed = 2 ** 32 - 1 - seed
    eval_env = gym.make(env_name)
    eval_env.seed(env_seed)

    avg_reward = 0.
    for _ in range(eval_episodes):
        state, done = eval_env.reset(), False
        while not done:
            action = policy.select_action(np.array(state), 'test')
            action = action[0]
            state, reward, done, _ = eval_env.step(action)
            avg_reward += reward

    avg_reward /= eval_episodes
    print("---------------------------------------")
    print("Evaluation over {} episodes: {:.3f}".format(eval_episodes, avg_reward))
    print("---------------------------------------")
    return avg_reward

if __name__ == "__main__":
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    env = gym.make("CartPole-v1")
    current_state = env.reset()
    state_dim = current_state.shape[0]
    n_actions = 2
    q_function = AdversarialDQN(state_dim=state_dim,
                                n_actions=n_actions,
                                device=device,
                                writer=None,
                                save_freq=100,
                                record_freq=100,
                                )
    start_learning = 100
    eval_freq = 100
    for t in range(20000):
        action = q_function.select_action(current_state, mode="train")
        action = action[0]
        next_state, reward, done, info = env.step(action)
        q_function.add_buffer(current_state, action, next_state, reward, done)
        if t == start_learning:
            print("start learning")
        if t > start_learning:
            q_function.train()
        current_state = next_state
        if done:
            current_state = env.reset()
        if t % eval_freq == 0:
            print('time steps: ',t)
            eval_policy(q_function, "CartPole-v1")


__author__ = 'Sergey Tomin'

import numpy as np
import matplotlib.pylab as plt
from scipy.optimize import brenth, root
from pylab import *
from numpy import linalg as LA

beta_x = 10.
alpha_x = 0.
gamma_x = (1 + alpha_x**2)/beta_x
beta_y = 5.
alpha_y = 0.
emit = 1.

def ellipse_trans(a11, a22, a33, a12 = 0, a13 = 0, a23 = 0, N = 100):
    """
    curved line of second order

    a11*x**2 + a22*y**2 + 2*a12*x*y + 2*a13*x + 2*a23*y + a33 = 0

    gamma*x**2 + 2*alpha*x*x' + beta*x'**2 = const
    """
    Delta = det([[a11, a12, a13],
                 [a12, a22, a23],
                 [a13, a23, a33]])
    M = array([[a11, a12],
               [a12, a22]])
    D = det(M)
    w, v = LA.eig(M)
    l1 = w[1]
    l2 = w[0]
    a = sqrt(-Delta/(l2*D))
    b = sqrt(-Delta/(l1*D))

    angle = arctan(2*a12/(a11 - a22))/2.
    t = linspace(0, 2*pi, num = N)
    xr = a*cos(t)
    yr = b*sin(t)
    #active axis transform - rotate
    x = xr*cos(angle) - yr*sin(angle)
    y = xr*sin(angle) + yr*cos(angle)
    return x, y

def ellipse_transform_SC(emit, tws):

    """
    curved line of second order

    a11*x**2 + a22*y**2 + 2*a12*x*y + 2*a13*x + 2*a23*y + a33 = 0

    gamma*x**2 + 2*alpha*x*x' + beta*x'**2 = const
    """
    #B1 = lambda a: beta_x * np.cos(a)**2 - 2 * alpha_x * np.cos(a) * np.sin(a) + gamma_x * np.sin(a)**2
    #A1 = lambda a: beta_x * np.sin(a)**2 + 2 * alpha_x * np.cos(a) * np.sin(a) + gamma_x * np.cos(a)**2
    B = lambda a: sin(a)**2 + (tws.beta_x*cos(a) - tws.alpha_x*sin(a))**2
    A = lambda a: cos(a)**2 + (tws.alpha_x*cos(a) + tws.beta_x*sin(a))**2
    # alternative way of finding angle
    """
    #f = lambda a: 2 * alpha_x * beta_x * np.cos(2 * a) + (beta_x**2 - 1 + alpha_x**2)* np.sin(2 *a)
    #f = lambda a: 2.*alpha_x* np.cos(a)**2 + 2 *beta_x* np.cos(a) *np.sin(a) - 2 *gamma_x* np.cos(a) *np.sin(a) - 2 *alpha_x*np.sin(a)**2
    #f =lambda a: (-2 * cos(a)* sin(a) + 2 * ( beta_x* cos(a) - alpha_x*sin(a)) *(alpha_x* cos(a) + beta_x* sin(a)))
    #sol = root(f, 0.1)
    #print "sol = ", sol.x
    #print arctan(2*alpha_x/(gamma_x-beta_x))/2.
    #angle = sol.x[0]
    """
    angle = arctan(2*tws.alpha_x/(tws.gamma_x-tws.beta_x))/2.

    a = np.sqrt(emit*tws.beta_x/A(angle))
    b = np.sqrt(emit*tws.beta_x/B(angle))
    #print a,b
    t = linspace(0, 2*pi, num = 100)
    xr = a*cos(t)
    yr = b*sin(t)

    x = xr*cos(angle) - yr*sin(angle)
    y = xr*sin(angle) + yr*cos(angle)
    return x,y

def phase_space_transform(x,y, tws):

    """
    curved line of second order

    a11*x**2 + a22*y**2 + 2*a12*x*y + 2*a13*x + 2*a23*y + a33 = 0

    gamma*x**2 + 2*alpha*x*x' + beta*x'**2 = const
    """

    angle = arctan(2*tws.alpha_x/(tws.gamma_x-tws.beta_x))/2.
    x = x*cos(angle) - y*sin(angle)
    y = x*sin(angle) + y*cos(angle)
    return x,y

if __name__ == "main":
    x = arange(-4,4, 0.01)
    y = lambda x: (np.sqrt(emit*beta_x - x*x) - alpha_x*x)/beta_x
    y2 = lambda x: (-np.sqrt(emit*beta_x - x*x) - alpha_x*x)/beta_x
    xnew, ynew = ellipse_trans(a11 = gamma_x, a22 = beta_x, a33 = - emit, a12 = alpha_x, a13 = 0, a23 = 0, N = 100)
    plt.plot(xnew, ynew)
    plot(x, y(x), x, y2(x))
    x1,y1 = ellipse_transform_SC(emit, tws)
    plot(x1,y1)
    plt.show()


# -*- coding: utf-8 -*-

import logging

from scrapy.exceptions import CloseSpider
from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError

from fooltrader import settings

logger = logging.getLogger(__name__)

# spider middleware
class FoolErrorMiddleware(HttpErrorMiddleware):
    def process_spider_exception(self, response, exception, spider):
        if isinstance(exception, HttpError):
            if response.status == 456:
                # response.meta['fool_blocked'] = True
                # return None
                raise CloseSpider('catch forbidden,close for a while')

# downloader middleware
class ForbiddenHandleMiddleware(object):
    forbidden_codes = (456, 403)

    def process_response(self, request, response, spider):
        proxy_count = len(settings.g_http_proxy_items)
        count = 0

        if request.meta.get('dont_proxy', False):
            return response
        if response.status in self.forbidden_codes:
            # 新浪财经
            if 'sina.com' in request.url:
                proxy_json = settings.g_http_proxy_items[count % proxy_count]
                count += 1
                if proxy_json['type'] == 'HTTPS':
                    proxy = 'https://{}:{}'.format(proxy_json['ip'], proxy_json['port'])
                elif proxy_json['type'] == 'HTTP':
                    proxy = 'http://{}:{}'.format(proxy_json['ip'], proxy_json['port'])
                request.meta['proxy'] = proxy
                logger.info("using")
                return request
        return response