import os
import re
import sys
import argparse
import subprocess
from time import sleep
from xml.etree import ElementTree

try:
    import youtube_dl

    ydl = youtube_dl.YoutubeDL(params={
        'noplaylist': True,
        'quiet': True,
    })
    ydl_selector = ydl.build_format_selector('bestaudio/best')
except ImportError:
    youtube_dl = None

NS = '{http://xspf.org/ns/0/}'
EXTS = ['mp3', 'ogg', 'opus', 'mp4', 'm4a', 'wav', 'flac', 'wma']
CHARS = re.compile('[^a-z0-9 ]')

__version__ = '0.0.0'

class DownloadPool(set):
    def log(self, s):
        print(s, file=sys.stderr, end='\r')

    def download(self, url, path):
        self.add(subprocess.Popen(['curl', url, '-s', '-o', path]))

    def poll(self):
        for p in list(self):
            if p.poll() is not None:
                self.remove(p)
        return len(self)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if exc_type is None:
            while self.poll():
                self.log('{} downloads still active'.format(len(self)))
                sleep(5)
        else:
            for p in self:
                p.kill()

def iter_files(folder):
    for dirpath, dirnames, filenames in os.walk(folder):
        for filename in filenames:
            if filename.rsplit('.', 1)[-1] in EXTS:
                yield dirpath, filename

def simplify(s):
    s = s.lower()
    s = s.encode('ascii', errors='replace').decode('ascii')
    return CHARS.sub('', s)

def find_by_title(title, fields, files):
    for dirpath, filename in files:
        path = os.path.join(dirpath, filename)
        if (
            simplify(title) in simplify(filename)
            and all(simplify(o) in simplify(path) for o in fields)
        ):
            return path

def search_youtube(q):
    # result expires in 6h
    try:
        info = ydl.extract_info('ytsearch:' + ' '.join(q), download=False)
    except youtube_dl.utils.DownloadError:
        return None, None
    if not info['entries']:
        return None, None
    info = info['entries'][0]
    try:
        _format = next(ydl_selector({'formats': info['formats']}))
    except StopIteration:
        return None, None
    filename = '{}-{}.{}'.format(
        info['title'].replace('/', '_'),
        info['id'],
        _format['ext'],
    )
    return _format['url'], filename

def iter_tracks(src):
    root = ElementTree.parse(src).getroot()
    for e in root.iter(NS + 'track'):
        track = {}
        for tag in ['location', 'title', 'creator', 'album', 'annotation']:
            field = e.find(NS + tag)
            if field is None:
                track[tag] = None
            else:
                track[tag] = field.text
        yield track

def hard_link(location, outdir):
    os.makedirs(outdir, exist_ok=True)
    filename = os.path.basename(location)
    path = os.path.join(outdir, filename)
    os.link(location, path)
    return path

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('src')
    parser.add_argument('folder')
    parser.add_argument('-Y', '--youtube', action='store_true')
    parser.add_argument('-O', '--outdir')
    return parser.parse_args()

def main():
    args = parse_args()
    files = list(iter_files(args.folder))

    if args.outdir:
        os.makedirs(args.outdir)

    with DownloadPool() as pool:
        for track in iter_tracks(args.src):
            location = track['location']

            if location is None:
                context_key = ['creator', 'annotation']
                context = [track[k] for k in context_key if track[k]]
                location = find_by_title(track['title'], context, files)
                if location and args.outdir:
                    location = hard_link(location, args.outdir)

            if location is None and args.youtube and youtube_dl:
                url, filename = search_youtube(
                    [q for q in track.values() if q]
                )
                if url is None:
                    location = None
                elif args.outdir:
                    location = os.path.join(args.outdir, filename)
                    pool.download(url, location)
                else:
                    location = url

            if location is None:
                s = ' - '.join('{}: {}'.format(k, v) for k, v in track.items())
                print('# Warning: ' + s)
            else:
                print(location)

if __name__ == '__main__':
    main()


from torch import nn

# class SELayer(nn.Module):
#     def __init__(self, channel, reduction=16):
#         super(SELayer, self).__init__()
#         self.avg_pool = nn.AdaptiveAvgPool2d(1)
#         self.fc = nn.Sequential(
#             nn.Linear(channel, channel // reduction, bias=False),
#             nn.ReLU(inplace=True),
#             nn.Linear(channel // reduction, channel, bias=False),
#             nn.Sigmoid()
#         )

#     def forward(self, x):
#         b, c, _, _ = x.size()
#         y = self.avg_pool(x).view(b, c)
#         y = self.fc(y).view(b, c, 1, 1)
#         return x * y.expand_as(x)

class SELayer(nn.Module):
    def __init__(self, channels, reduction):
        super(SELayer, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
                             padding=0)
        self.relu = nn.ReLU(inplace=True)
        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
                             padding=0)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        module_input = x
        x = self.avg_pool(x)
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        x = self.sigmoid(x)
        return module_input * x

'''
weight:
layer1.0.conv1.weight
layer1.0.bn1.weight
layer1.0.bn1.bias
layer1.0.bn1.running_mean
layer1.0.bn1.running_var
layer1.0.conv2.weight
layer1.0.bn2.weight
layer1.0.bn2.bias
layer1.0.bn2.running_mean
layer1.0.bn2.running_var
layer1.0.conv3.weight
layer1.0.bn3.weight
layer1.0.bn3.bias
layer1.0.bn3.running_mean
layer1.0.bn3.running_var

layer1.0.se_module.fc1.weight / torch.Size([16, 256, 1, 1])
layer1.0.se_module.fc1.bias
layer1.0.se_module.fc2.weight / torch.Size([256, 16, 1, 1])
layer1.0.se_module.fc2.bias

layer1.0.downsample.0.weight
layer1.0.downsample.1.weight
layer1.0.downsample.1.bias
layer1.0.downsample.1.running_mean
layer1.0.downsample.1.running_var

====================

model
layer1.0.conv1.weight
layer1.0.bn1.weight
layer1.0.bn1.bias
layer1.0.bn1.running_mean
layer1.0.bn1.running_var
layer1.0.bn1.num_batches_tracked
layer1.0.conv2.weight
layer1.0.bn2.weight
layer1.0.bn2.bias
layer1.0.bn2.running_mean
layer1.0.bn2.running_var
layer1.0.bn2.num_batches_tracked
layer1.0.conv3.weight
layer1.0.bn3.weight
layer1.0.bn3.bias
layer1.0.bn3.running_mean
layer1.0.bn3.running_var
layer1.0.bn3.num_batches_tracked

layer1.0.se.fc.0.weight / torch.Size([16, 256])
layer1.0.se.fc.2.weight

layer1.0.downsample.0.weight
layer1.0.downsample.1.weight
layer1.0.downsample.1.bias
layer1.0.downsample.1.running_mean
layer1.0.downsample.1.running_var
layer1.0.downsample.1.num_batches_tracked
'''


import os
import re
import mmap
from pymongo import MongoClient

import config

client = MongoClient()
db = client[config.MONGO_DATABASE]
sources_collection = db[config.MONGO_COLLECTION_SOURCES]
countries_collection = db[config.MONGO_COLLECTION_COUNTRIES]

minerals_regex = re.compile(bytes("(%s)" % "|".join(config.MINERALS), 'utf-8'), re.IGNORECASE | re.MULTILINE)

for source in sources_collection.find({'newest_xls_local_csv_directory': {'$exists': 1}}):
    print("Processing source %s..." % source['_id'])
    for table in os.listdir(source['newest_xls_local_csv_directory']):
        print("  └ %s..." % table)
        with open(os.path.join(source['newest_xls_local_csv_directory'], table), 'rb', 0) as file, \
                mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as file_mmap:
            mineral_matches = [mineral.decode('utf-8').lower() for mineral in minerals_regex.findall(file_mmap)]
            mineral_matches = list(dict.fromkeys(mineral_matches))  # deduplicate
            print('   ', mineral_matches)
            sources_collection.update_one(
                {'_id': source['_id']},
                {'$set': {'minerals': mineral_matches}}
            )
            countries_collection.update_many(
                {'_id': {'$in': source['used_by']}},
                {'$addToSet': {'minerals': {'$each': mineral_matches}}}
            )


import streamlit as st
import pandas as pd

st.set_page_config(layout="wide")

col1, col2, col3 = st.columns([1, 2, 0.25])

df = pd.read_csv("data.csv", sep=";")

with col1:
    st.image("images/photo.png", width=300)

with col2:
    st.title("Başak Dilara Çevik")
    content1 = """After graduating from the department of economics,
      I started to pursue my master's degree in economics and had the opportunity to develop
       myself in the research area.
        Currently, I am conducting a thesis study to accelerate the adoption of electric vehicles
         in Turkey within the scope of sustainability. In this process,
          I significantly improved my knowledge of game development with Unity, project management,
          and game art with the help of Oyun ve Uygulama Akademisi.
          Besides all these, I have been actively developing myself in the fields of data science and machine learning
          for the last three years, participating in various events and communities. :)"""
    st.info(content1)

content2 = """This is my portfolio. You can navigate between the bars to see my projects."""
st.subheader(content2)

tab1, tab2, tab3, tab4, tab5, tab6,\
tab7, tab8, tab9, tab10, \
tab11, tab12, \
tab13, tab14, tab15 = st.tabs(["To-Do App", "Portfolio", "PDF Generation", "Excel Invoices to PDF",
                                  "Email News", "Weather API", "Book Sentiment Analysis",
                                  "Email Webcam Object Detection", "Web Scraping Musical Events",
                                  "Hotel Booking", "Super Mario Game Code Inspection",
                                  "Student Management System", "Chatbot", "Astronomy Image of the Day", "Weather Forecast"])

#if you prefer columnwise rather than tabwise
# for index, row in df.iterrows():
#     st.header(row["title"])
#     st.write(row["description"])
#     st.image("images/" + row["image"])
#     st.write("[Source Code](https://badice.com.tr)")

with tab1:
   st.header("To-Do App")
   st.write(df["description"][0])
   st.image("images/1.png", width=500)
   st.write(f"[Source Code]({df['url'][0]})")

with tab2:
   st.header("Portfolio")
   st.write(df["description"][1])
   st.image("images/2.png", width=500)
   st.write(f"[Source Code]({df['url'][1]})")

with tab3:
   st.header("Excel Invoices to PDF")
   st.write(df["description"][2])
   st.image("images/3.png", width=500)
   st.write(f"[Source Code]({df['url'][2]})")

with tab4:
    st.header("")
    st.write(df["description"][3])
    st.image("images/4.png", width=500)
    st.write(f"[Source Code]({df['url'][3]})")

with tab5:
    st.header("")
    st.write(df["description"][4])
    st.image("images/5.png", width=500)
    st.write(f"[Source Code]({df['url'][4]})")

with tab6:
    st.header("")
    st.write(df["description"][5])
    st.image("images/6.png", width=500)
    st.write(f"[Source Code]({df['url'][5]})")

with tab7:
    st.header("")
    st.write(df["description"][6])
    st.image("images/7.png", width=500)
    st.write(f"[Source Code]({df['url'][6]})")

with tab8:
    st.header("")
    st.write(df["description"][7])
    st.image("images/8.png", width=500)
    st.write(f"[Source Code]({df['url'][7]})")

with tab9:
    st.header("")
    st.write(df["description"][8])
    st.image("images/9.png", width=500)
    st.write(f"[Source Code]({df['url'][8]})")

with tab10:
    st.header("")
    st.write(df["description"][9])
    st.image("images/10.png", width=500)
    st.write(f"[Source Code]({df['url'][9]})")

with tab11:
    st.header("")
    st.write(df["description"][10])
    st.image("images/11.png", width=500)
    st.write(f"[Source Code]({df['url'][10]})")

with tab12:
    st.header("")
    st.write(df["description"][11])
    st.image("images/12.png", width=500)
    st.write(f"[Source Code]({df['url'][11]})")

with tab13:
    st.header("")
    st.write(df["description"][12])
    st.image("images/13.png", width=500)
    st.write(f"[Source Code]({df['url'][12]})")

with tab14:
    st.header("")
    st.write(df["description"][13])
    st.image("images/14.png", width=500)
    st.write(f"[Streamlit Website]({df['url'][13]})")

with tab15:
    st.header("")
    st.write(df["description"][16])
    st.image("images/16.png", width=500)
    st.write(f"[Streamlit Website]({df['url'][16]})")


from typing import Optional, Dict

from allennlp.data import Vocabulary
from allennlp.modules import FeedForward, Seq2SeqEncoder, TextFieldEmbedder
from allennlp.modules import InputVariationalDropout, TimeDistributed
from allennlp.modules.attention import DotProductAttention
from allennlp.models.model import Model
from allennlp.nn import InitializerApplicator, RegularizerApplicator
from allennlp.nn import util
from allennlp.training.metrics import CategoricalAccuracy, F1Measure
import numpy as np
import torch
import torch.nn.functional as F
from torch.nn import Linear, Dropout
from torch.nn.parameter import Parameter

@Model.register("attention_dialect_classifier")
class AttentionDialectClassifier(Model):
    def __init__(self,
                 vocab: Vocabulary,
                 text_field_embedder: TextFieldEmbedder,
                 text_encoder: Seq2SeqEncoder,
                 classifier_feedforward: Optional[FeedForward] = None,
                 dropout: Optional[float] = 0.0,
                 code_switching_regularizer: Optional[float] = 0.0,
                 bivalency_regularizer: Optional[float] = 0.0,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None) -> None:
        '''
        :param dropout: The amount of dropout to apply. Dropout is applied
                        after each non-linear layer and the word embeddings
                        lookup. Two types of dropout are applied, variational
                        dropout is applied if the input is to the dropout is
                        a sequence of vectors (each vector in the sequence
                        representing a word), and normal dropout if the input
                        is a vector.
        :param code_switching_regularizer: The weight associated to the code
                                           switching lexicon regulisation the
                                           lower the less affect it has. This
                                           requires that the dataset reader is
                                           going to supply the code switching
                                           arrays for the forward function of
                                           this class. If set a good values is
                                           0.001
        :param bivalency_regularizer: The weight associated to the bivalency
                                      regulisation the lower the less affect it
                                      has. This requires that the dataset
                                      reader is going to supply the bivalency
                                      arrays for the forward function of this
                                      class.
        '''
        super().__init__(vocab, regularizer)
        self._naive_dropout = Dropout(dropout)
        self._variational_dropout = InputVariationalDropout(dropout)

        self.text_field_embedder = text_field_embedder
        self.num_classes = self.vocab.get_vocab_size("labels")
        self.text_encoder = text_encoder
        text_encoder_dim = text_encoder.get_output_dim()
        # Attention parameters
        self.project_encoded_text = TimeDistributed(Linear(text_encoder_dim,
                                                           text_encoder_dim))
        self.attention_vector = Parameter(torch.Tensor(text_encoder_dim))
        self.reset_parameters()
        self.attention_layer = DotProductAttention(normalize=True)

        self.classifier_feedforward = classifier_feedforward
        output_dim = text_encoder_dim
        if classifier_feedforward:
            output_dim = classifier_feedforward.get_output_dim()
        self.label_projection = Linear(output_dim, self.num_classes)
        self.metrics = {
                "accuracy": CategoricalAccuracy()
        }
        self.code_switching_regularizer = code_switching_regularizer
        self.bivalency_regularizer = bivalency_regularizer
        self.loss = torch.nn.CrossEntropyLoss()
        initializer(self)

    def reset_parameters(self):
        '''
        Intitalises the attnention vector
        '''
        torch.nn.init.uniform_(self.attention_vector, -0.01, 0.01)

    def lexicon_loss(self, lexicon_array: torch.Tensor,
                     attention_weights: torch.Tensor,
                     text_mask: torch.Tensor) -> float:
        '''
        :param lexicon_array: Tensor of shape [batch, sequence_length] that
                              contains either 0, -1, or 1 values where -1
                              has to be for the whole sample in the batch
                              which would indicate not to perform attention
                              regulisation else 0 and 1 indicate important words
                              in the sequence that should be attended to.
        :param attention_weights: Tensor of shape [batch, sequence_length]
                                  that contains weights associating to
                                  the importance of the words in the sequence.
        :param text_mask: Tensor of shape [batch, sequence_length]. Mask
                          denoting whether there is a word or not at that
                          position.
        :returns: The difference between the lexicon_array denotating words
                  that should important to the network and those that the
                  network think are important via it's attention mechansim.
                  The difference is computed via cross entropy.
        '''
        # Mask is required to not add loss when the label is MSA
        lexicon_mask = lexicon_array.sum(1)
        lexicon_mask = (lexicon_mask >= 0).float()
        # Calculating the lexicon loss
        lexicon_attention_weights = attention_weights.squeeze()
        lexicon_softmax = util.masked_softmax(lexicon_array, text_mask)
        # This is to stop NANS from performing log on 0.
        lexicon_attention_weights = lexicon_attention_weights + 1e-8
        # Perform cross entropy like in the paper to get a loss function
        # that represents disagreement between the two vectors
        lexicon_loss = lexicon_softmax * torch.log(lexicon_attention_weights)
        lexicon_loss = lexicon_loss * text_mask.float()
        lexicon_loss = lexicon_loss.abs()
        lexicon_loss = lexicon_loss.sum(1)

        lexicon_loss = lexicon_loss * lexicon_mask
        lexicon_loss = lexicon_loss.sum()
        return lexicon_loss

    def forward(self,
                text: Dict[str, torch.LongTensor],
                label: torch.LongTensor = None,
                code_switching_array: torch.Tensor = None,
                bivalency_array: torch.Tensor = None
                ) -> Dict[str, torch.Tensor]:
        # Embed text
        embedded_text = self.text_field_embedder(text)
        embedded_text = self._variational_dropout(embedded_text)
        text_mask = util.get_text_field_mask(text)
        # Encode text
        encoded_text = self.text_encoder(embedded_text, text_mask)
        encoded_text = self._variational_dropout(encoded_text)
        #
        # ATTENTION
        #
        # Project each hidden timestep into a new hidden space
        # before attention
        projected_encoding = self.project_encoded_text(encoded_text)
        projected_encoding = torch.tanh(projected_encoding)
        projected_encoding = self._variational_dropout(projected_encoding)
        # Apply attention over the project encoded text
        batch_size = text_mask.shape[0]
        attention_vector = self.attention_vector.unsqueeze(0).expand(batch_size, -1)
        attention_weights = self.attention_layer(attention_vector,
                                                 projected_encoding, text_mask)
        attention_weights = attention_weights.unsqueeze(-1)
        weighted_encoded_text = attention_weights * encoded_text
        weighted_encoded_vec = weighted_encoded_text.sum(1)
        weighted_encoded_vec = self._naive_dropout(weighted_encoded_vec)

        if self.classifier_feedforward:
            weighted_encoded_vec = self.classifier_feedforward(weighted_encoded_vec)
        logits = self.label_projection(weighted_encoded_vec)
        class_probabilities = F.softmax(logits, dim=-1)

        output_dict = {"class_probabilities": class_probabilities}

        if label is not None:

            loss = self.loss(logits, label)

            if code_switching_array is not None and self.training:
                code_switch_loss = self.lexicon_loss(code_switching_array,
                                                     attention_weights, text_mask)
                loss = loss + (self.code_switching_regularizer * code_switch_loss)
            if bivalency_array is not None and self.training:
                bivalency_loss = self.lexicon_loss(bivalency_array,
                                                   attention_weights, text_mask)
                loss = loss + (self.bivalency_regularizer * bivalency_loss)

            for metric in self.metrics.values():
                metric(logits, label)

            output_dict["loss"] = loss

        return output_dict

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        return {metric_name: metric.get_metric(reset)
                for metric_name, metric in self.metrics.items()}

    def decode(self, output_dict: Dict[str, torch.Tensor]
               ) -> Dict[str, torch.Tensor]:
        predictions = output_dict['class_probabilities'].cpu().data.numpy()
        argmax_indices = np.argmax(predictions, axis=-1)
        labels = [self.vocab.get_token_from_index(x, namespace="labels")
                  for x in argmax_indices]
        output_dict['label'] = labels
        return output_dict


"""
The GripNet proposed in the `"GripNet: Graph Information Propagation on Supergraph for Heterogeneous Graphs"
    <https://doi.org/10.1016/j.patcog.2022.108973>`_ (PatternRecognit 2022) paper, which is an efficient
    framework to learn node representations on heterogeneous graphs for the downstream link prediction,
    node classification, and visualization. The code is based on the https://github.com/NYXFLOWER/GripNet.
"""

import logging
from typing import Dict

import torch
import torch.nn.functional as F
from torch.nn import Module

from kale.embed.gcn import GCNEncoderLayer, RGCNEncoderLayer
from kale.prepdata.supergraph_construct import SuperGraph, SuperVertex, SuperVertexParaSetting

class GripNetInternalModule(Module):
    r"""
    The internal module of a supervertex, which is composed of an internal feature layer and multiple internal
    aggregation layers.

    Args:
        in_channels (int): the dimension of node features on this supervertex.
        num_edge_type (int): the number of edge types on this supervertex.
        start_supervertex (bool): whether this supervertex is a start supervertex on the supergraph.
        setting (SuperVertexParaSetting): supervertex parameter settings.
    """

    def __init__(
        self, in_channels: int, num_edge_type: int, start_supervertex: bool, setting: SuperVertexParaSetting
    ) -> None:
        super(GripNetInternalModule, self).__init__()
        # in and out dimension
        self.in_channels = in_channels
        self.out_channels = setting.inter_agg_channels_list[-1]

        self.num_edge_type = num_edge_type
        self.multirelational = 1 if num_edge_type > 1 else 0
        self.start_supervertex = start_supervertex
        self.setting = setting

        self.__init_inter_feat_layer__()
        self.__init_inter_agg_layer__()

    def __init_inter_feat_layer__(self):
        """internal feature layer"""

        self.embedding = torch.nn.Parameter(torch.Tensor(self.in_channels, self.setting.inter_feat_channels))
        self.embedding.requires_grad = True

        # reset parameters to be normally distributed
        self.embedding.data.normal_()

    def __init_inter_agg_layer__(self):
        """internal aggregation layers"""

        # compute the dim of input of the first internal aggregation layer
        self.in_agg_channels = self.setting.inter_feat_channels
        if not self.start_supervertex:
            if self.setting.mode == "cat":
                if not self.setting.exter_agg_channels_dict:
                    error_msg = "`exter_agg_channels_dict` is not set."
                    logging.error(error_msg)
                    raise ValueError(error_msg)
                self.in_agg_channels += sum(self.setting.exter_agg_channels_dict.values())
            elif self.setting.mode == "add":
                tmp = set([self.in_agg_channels] + list(self.setting.exter_agg_channels_dict.values()))
                if len(tmp) != 1:
                    error_msg = "`in_agg_channels` should be the same as any element in `exter_agg_channels_dict`."
                    logging.error(error_msg)
                    raise ValueError(error_msg)
            else:
                error_msg = "`mode` value is invalid. Use 'cat' or 'add'."
                logging.error(error_msg)
                raise ValueError(error_msg)

        # create and initialize the internal aggregation layers
        self.num_inter_agg_layer = len(self.setting.inter_agg_channels_list)
        tmp_dim = [self.in_agg_channels] + self.setting.inter_agg_channels_list

        if self.setting.concat_output:
            self.out_channels = sum(tmp_dim)

        if self.multirelational:
            # using RGCN if there are multiple edge types
            after_relu = [False if i == 0 else True for i in range(self.num_inter_agg_layer)]
            self.inter_agg_layers = torch.nn.ModuleList(
                [
                    RGCNEncoderLayer(
                        tmp_dim[i], tmp_dim[i + 1], self.num_edge_type, self.setting.num_bases, after_relu[i]
                    )
                    for i in range(self.num_inter_agg_layer)
                ]
            )
        else:
            # using GCN if there is only one edge type
            self.inter_agg_layers = torch.nn.ModuleList(
                [GCNEncoderLayer(tmp_dim[i], tmp_dim[i + 1], cached=True) for i in range(self.num_inter_agg_layer)]
            )

    def forward(
        self,
        x: torch.Tensor,
        edge_index: torch.Tensor,
        edge_type: torch.Tensor = None,
        range_list: torch.Tensor = None,
        edge_weight: torch.Tensor = None,
    ) -> torch.Tensor:
        r"""
        Args:
            x (torch.Tensor): the input node feature embedding.
            edge_index (torch.Tensor): edge index in COO format with shape [2, #edges].
            edge_type (torch.Tensor, optional): one-dimensional relation type for each edge, indexed from 0.
                Defaults to None.
            range_list (torch.Tensor, optional): The index range list of each edge type with shape [num_types, 2].
                Defaults to None.
            edge_weight (torch.Tensor, optional): one-dimensional weight for each edge. Defaults to None.

        Note: The internal feature layer is computed in the `forward` function of GripNet class. If the supervertex
        is not a start supervertex, `x` should be the sum or concat of the outputs of the internal feature
        layer and all external aggregation layers.
        """

        if self.setting.concat_output:
            tmp = []
            tmp.append(x)

        if self.multirelational:
            if edge_type is None or range_list is None:
                error_msg = "`edge_type` and `range_list` are not set."
                logging.error(error_msg)
                raise ValueError(error_msg)

        # internal feature aggregation layers
        for net in self.inter_agg_layers[:-1]:
            x = net(x, edge_index, edge_type, range_list) if self.multirelational else net(x, edge_index, edge_weight)
            x = F.relu(x, inplace=True)
            if self.setting.concat_output:
                tmp.append(x)

        x = (
            self.inter_agg_layers[-1](x, edge_index, edge_type, range_list)
            if self.multirelational
            else self.inter_agg_layers[-1](x, edge_index, edge_weight)
        )

        x = F.relu(x, inplace=True)
        if self.setting.concat_output:
            tmp.append(x)
            x = torch.cat(tmp, dim=1)

        return x

    def __repr__(self):
        tmp = [f"\n    ({i}): {l}" for i, l in enumerate(self.inter_agg_layers)]

        return (
            "{}: ModuleList(\n  (0): InternalFeatureLayer: Embedding({}, {})\n  "
            "(1): InternalFeatureAggregationModule: ModuleList({}\n  )\n)"
        ).format(self.__class__.__name__, self.in_channels, self.setting.inter_feat_channels, "".join(tmp))

class GripNetExternalModule(Module):
    """The internal module of a supervertex, which is an external feature layer.

    Args:
        in_channels (int): Size of each input sample. In GripNet, it should be the dimension of the output embedding of
        the corresponding parent supervertex.
        out_channels (int): Size of each output sample. In GripNet, it is the dimension of the output embedding of
            the supervertex.
        num_out_node (int): the number of output nodes.
    """

    def __init__(self, in_channels: int, out_channels: int, num_out_node: int) -> None:
        super(GripNetExternalModule, self).__init__()

        self.in_channels = in_channels
        self.out_channels = out_channels
        self.num_out_node = num_out_node

        self.exter_agg_layer = GCNEncoderLayer(in_channels, out_channels, cached=True)

    def forward(self, x: torch.Tensor, edge_index: torch.Tensor, edge_weight: torch.Tensor = None, use_relu=True):
        """
        Args:
            x (torch.Tensor): the input node feature embedding.
            edge_index (torch.Tensor): edge index in COO format with shape [2, #edges].
            edge_weight (torch.Tensor, optional): one-dimensional weight for each edge. Defaults to None.
            use_relu (bool, optional): whether to use ReLU before returning node feature embeddings. Defaults to True.
        """

        n_source, n_feat = x.shape
        bigraph_edge_index = edge_index + 0
        bigraph_edge_index[1, :] += n_source

        x = torch.cat([x, torch.zeros((self.num_out_node, n_feat)).to(x.device)], dim=0)
        x = self.exter_agg_layer(x, bigraph_edge_index, edge_weight)[n_source:, :]

        if use_relu:
            x = F.relu(x, inplace=True)

        return x

    def __repr__(self):
        return "{}: {}".format(self.__class__.__name__, self.exter_agg_layer.__repr__())

class GripNet(Module):
    r"""The GripNet model.

    Args:
        supergraph (SuperGraph): the supergraph.

    Reference:
        Xu, H., Sang, S., Bai, P., Li, R., Yang, L. and Lu, H., 2022. GripNet: Graph Information
        Propagation on Supergraph for Heterogeneous Graphs. Pattern Recognition, p.108973.
    """

    def __init__(self, supergraph: SuperGraph) -> None:
        super(GripNet, self).__init__()

        self.supergraph = supergraph
        self.task_supervertex_name = supergraph.topological_order[-1]

        self.out_embed_dict: Dict[str, torch.Tensor] = {}

        self.__check_supergraph__()
        self.__init_supervertex_module_dict__()

        self.out_channels = self.__get_out_channels__()

    def __check_supergraph__(self) -> None:
        """check whether the input supergraph has parameter settings"""

        if self.supergraph.supervertex_setting_dict is None:
            error_msg = "`supervertex_setting_dict` is not set."
            logging.error(error_msg)
            raise ValueError(error_msg)

    def __init_supervertex_module_dict__(self):
        self.supervertex_module_dict: Dict[str, torch.nn.ModuleList] = {}
        for supervertex_name in self.supergraph.topological_order:
            supervertex = self.supergraph.supervertex_dict[supervertex_name]
            setting = self.supergraph.supervertex_setting_dict[supervertex_name]
            self.__init_module_supervertex__(supervertex, setting)

    def __init_module_supervertex__(self, supervertex: SuperVertex, setting: SuperVertexParaSetting):
        module_list = torch.nn.ModuleList()

        if not supervertex.start_supervertex:
            # add the external modules from all parent supervertices
            for in_name in supervertex.in_supervertex_list:
                in_channels = self.supervertex_module_dict[in_name][-1].out_channels

                if setting.exter_agg_channels_dict is None:
                    error_msg = "`exter_agg_channels_dict` is not set."
                    logging.error(error_msg)
                    raise ValueError(error_msg)

                out_channels = setting.exter_agg_channels_dict[in_name]
                module_list.append(GripNetExternalModule(in_channels, out_channels, supervertex.num_node))

        # add the internal module
        module_list.append(
            GripNetInternalModule(
                supervertex.num_node_feat, supervertex.num_edge_type, supervertex.start_supervertex, setting
            )
        )

        self.supervertex_module_dict[supervertex.name] = module_list

    def __get_out_channels__(self):
        task_supervertex = self.supergraph.topological_order[-1]
        return self.supervertex_module_dict[task_supervertex][-1].out_channels

    def forward(self):
        if self.supergraph.supervertex_setting_dict is None:
            error_msg = "`supervertex_setting_dict` is not set."
            logging.error(error_msg)
            raise ValueError(error_msg)

        for supervertex_name in self.supergraph.topological_order:
            mode = self.supergraph.supervertex_setting_dict[supervertex_name].mode
            self.__forward_supervertex__(supervertex_name, mode)

        return self.out_embed_dict[self.task_supervertex_name]

    def __forward_supervertex__(self, supervertex_name: str, mode: str):
        supervertex = self.supergraph.supervertex_dict[supervertex_name]
        model = self.supervertex_module_dict[supervertex_name]

        # internal feature layer
        x = torch.matmul(supervertex.node_feat, model[-1].embedding)

        # external feature aggregation layers
        if mode == "add":
            for idx in range(len(supervertex.in_supervertex_list)):
                parent_name = supervertex.in_supervertex_list[idx]
                parent_x = self.out_embed_dict[parent_name]
                superedge = self.supergraph.superedge_dict[(parent_name, supervertex_name)]

                x += model[idx](parent_x, superedge.edge_index, superedge.edge_weight)
        else:
            tmp = [x]
            for idx in range(len(supervertex.in_supervertex_list)):
                parent_name = supervertex.in_supervertex_list[idx]
                parent_x = self.out_embed_dict[parent_name]
                superedge = self.supergraph.superedge_dict[(parent_name, supervertex_name)]

                tmp.append(model[idx](parent_x, superedge.edge_index, superedge.edge_weight))
            x = torch.cat(tmp, dim=1)

        # internal feature aggregation layers
        if supervertex.num_edge_type > 1:
            x = model[-1](
                x, supervertex.edge_index, supervertex.edge_type, supervertex.range_list, supervertex.edge_weight
            )
        else:
            x = model[-1](x, supervertex.edge_index, supervertex.edge_weight)

        self.out_embed_dict[supervertex_name] = x

    def __repr__(self):
        return "{}: ModuleDict(\n{})".format(self.__class__.__name__, self.supervertex_module_dict)


from sklearn.metrics import f1_score
from simpletransformers.classification import MultiLabelClassificationModel

import sys
import os

import pandas as pd
import datetime
import csv
import numpy as np

root_path = os.path.dirname(os.path.dirname(
    os.path.dirname(os.path.abspath(__file__))))

prep_path = os.path.join(root_path, 'preprocess')
sys.path.append(prep_path)

split_path = os.path.join(root_path, 'dataset/split')

curr_path = os.path.dirname(os.path.abspath(__file__))

from reader import Reader

def f1_evaluate(true, pred):
    for p in pred:
        for i in range(len(p)):
            if p[i] >= 0.5:
                p[i] = 1
            else:
                p[i] = 0

    score = f1_score(true, pred, average = 'macro')
    label = f1_score(true, pred, average = None)
    print(score)
    print(label)

    return score

class BERTMultilabel:
    def __init__(self):
        self.dummy = None
        self.reader = Reader()

    def train_model(self, training_data, training_args, base_model_path, test_data):
        model = MultiLabelClassificationModel(
            "bert",
            base_model_path,
            num_labels=8,
            use_cuda = False, # Highly recommended to set use_cuda = True to ultilize GPU (if available) for training
            args = training_args,
        )

        temp = [self.reader.assign_label(s) for s in training_data]

        test_data = [self.reader.assign_label(s) for s in test_data]
        test_text = [s[0] for s in test_data]
        test_label = [s[1] for s in test_data]

        eval_df = pd.DataFrame(test_data, columns=['text', 'labels'])

        model.train_model(pd.DataFrame(temp, columns=['text', 'labels']), eval_df = eval_df, f1_macro = f1_evaluate)

        return model

    def test_model(self, model, test_data):
        if model == None:
            print('No model found')
            return

        test_data = [self.reader.assign_label(s) for s in test_data]
        test_text = [s[0] for s in test_data]
        test_label = [s[1] for s in test_data]

        predictions, dummy = model.predict(test_text)
        pred = pd.DataFrame(predictions)
        pd_label = pd.DataFrame(test_label)

        np.set_printoptions(formatter={'float': '{: 0.3f}'.format})

        f1_macro = f1_score(pd_label, pred, average='macro')
        f1_all = f1_score(pd_label, pred, average=None)

        print(f1_macro)
        print(f1_all)

        return f1_macro, f1_all

# # Training args
# args = {"reprocess_input_data": True,
# "overwrite_output_dir": True,
# "num_train_epochs": 30,
# 'fp16': False,
# "use_early_stopping": True,
# 'learning_rate': 4e-5,
# 'evaluate_during_training' : True,
# 'early_stopping_metric' : 'f1_macro',
# 'early_stopping_metric_minimize': False,
# 'save_model_every_epoch' : False,
# 'train_batch_size' : 8
# }

# # Read the splits

# r = Reader()

# split_0 = r.read_from_split(split_path + '/split_0.tsv')
# split_1 = r.read_from_split(split_path + '/split_1.tsv')

# base_model_path = 'bert-large-uncased'

# bm = BERTMultilabel()

# # Train 01 base model
# train = split_0
# test = split_1

# model = bm.train_model(train, args, base_model_path, test)


import requests
kv={'q':'python'}
url='http://www.so.com/s'
try:
    r=requests.get(url,params=kv)
    r.raise_for_status()
    r.encoding=r.apparent_encoding
    print(r.text)
except:
    print('爬虫失败')
#一堆乱码
'''
r=requests.get(url,params=kv)
r.encoding=r.apparent_encoding
print(r.request.url)
#莫名其妙 的百度的搜索关键词这样用不行
'''


import tensorflow as tf
import numpy as np
import pygame

#Checkpoint System
checkpoint_path="pong/pong.ckpt"
cp_callback=tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,save_weights_only=True,verbose=1)

#AI
    #model
model = tf.keras.Sequential([
    tf.keras.layers.Input(shape=(5)),
    tf.keras.layers.Dense(5,activation='relu'),
    tf.keras.layers.Dense(2)
])

model.compile(optimizer='adam',
    #loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    loss='categorical_crossentropy',
    metrics=['accuracy'])
#load Checkpoint
model.load_weights(checkpoint_path)

inputs=[]
outputs=[]

def resArrays():
    inputs=[]
    outputs=[]
counter = 0

bs = 5
#Screen and drawning
size = width,height = 400,400
    #init
pygame.init()
screen = pygame.display.set_mode((int(width),int(height)))

    #DrawningColor
c_open= (   255,255,255)
c_off = (   0,  0,  0)

def drawRect(x,y,width,height):
    pygame.draw.rect(screen,c_open,pygame.Rect(int(x),int(y),int(width),int(height)))

#Object
class Object:
    def __init__(self,startX=45,startY=45,startSX=10,startSY=10,startSpeed = 1):
        self.pos,self.size = [startX,startY],[startSX,startSY]
        self.speed = startSpeed

    def getCol(self, cpX,cpY):
        return (self.pos[0]<cpX) and (self.pos[0]+self.size[0]>cpX) and (self.pos[1]<cpY) and (self.pos[1]+self.size[1]>cpY)

    def getColBlock(self,pos,size):
        return self.getCol(pos[0],pos[1]) or self.getCol(pos[0]+size[0],pos[1]) or self.getCol(pos[0],pos[1]+size[1]) or self.getCol(pos[0]+size[0],pos[1]+size[1])

    def print(self):
        drawRect(self.pos[0],self.pos[1],self.size[0],self.size[1])

    def move(self,mX,mY):
        self.pos[0]+=int(mX*self.speed)
        self.pos[1]+=int(mY*self.speed)

#Control
control1=[0.0,0.0]
control2=[0.0,0.0]
score=[0,0]

player1 = Object(0,40,10,100,6)
player2 = Object(390,40,10,100,6)
ball    = Object(200,200,10,10,2)
ballV   = [1.0,0.0]

def setVector(l):
    ballV[1]+=0.03*l

#System
clock = pygame.time.Clock()
isOpen = True
while isOpen:
    #Controls
    for event in pygame.event.get():
        if event.type == pygame.QUIT: isOpen=False

    keys=pygame.key.get_pressed()
    if(keys[pygame.K_w]):control1[1]=1.0
    else:control1[1]=0.0
    if(keys[pygame.K_s]):control1[0]=1.0
    else:control1[0]=0.0

    if(keys[pygame.K_o]):control2[1]=1.0
    else:control2[1]=0.0
    if(keys[pygame.K_l]):control2[0]=1.0
    else:control2[0]=0.0
    #control2 = [0.0,0.0]
    inputToAI=[ball.pos[0]/400,ball.pos[1]/400,ballV[0]/5,ballV[1]/5,player2.pos[1]/400]
    AIControls=model.predict([inputToAI])
    control2=AIControls[0]
    print("AI:",AIControls)

    #Train Values
    control=np.array(control1)
    values=np.array ([((400-ball.pos[0])/400),ball.pos[1]/400,((5-ballV[0])/5),ballV[1]/5,player1.pos[1]/400])
    inputs.append(values)
    outputs.append(control)

    #inputs=np.insert(inputs,values,axis=2)
    #outputs=np.instert(outputs,control,axis=2)

    #Move
    moveLimit = 0.3
    if(control1[0]>moveLimit):player1.move(0,1)
    if(control1[1]>moveLimit):player1.move(0,-1)

    if(control2[0]>moveLimit):player2.move(0,1)
    if(control2[1]>moveLimit):player2.move(0,-1)

    ball.move(ballV[0],ballV[1])

    #Collision
    if(ball.pos[1]<0) : ballV[1]*=-1
    elif(ball.pos[1]+ball.size[1]>400):ballV[1]*=-1

    if(player1.getColBlock(ball.pos, ball.size)) :
        ballV[0]*=-1
        setVector((ball.pos[1]+(ball.size[1]/2))-(player1.pos[1]+(player1.size[1]/2)))
    elif(player2.getColBlock(ball.pos, ball.size)) :
        ballV[0]*=-1
        setVector((ball.pos[1]+(ball.size[1]/2))-(player2.pos[1]+(player2.size[1]/2)))
        resArrays()

    if(player1.pos[1]<0):player1.pos[1]=0
    elif(player1.pos[1]+player1.size[1]>400):player1.pos[1]=300
    if(player2.pos[1]<0):player2.pos[1]=0
    elif(player2.pos[1]+player2.size[1]>400):player2.pos[1]=300

        #Win
    if(ball.pos[0]+ball.size[0]>400):
        ball.pos=[200,200]
        score[0]+=1
        ballV=[-1.0,0.0]
        print("P1:",score[0]," P2:",score[1])
        #Train Model
            #wu
        inputsT=np.vstack(inputs)
        outputsT=np.vstack(outputs)
        print("I:",inputsT)
        print("O:",outputsT)
        model.fit(inputsT,outputsT,callbacks=[cp_callback])
        #Reset Inputs and Outputs
        resArrays()
        player1.pos=[0,150]
        player2.pos=[390,150]
    elif(ball.pos[0]<0):
        ball.pos=[200,200]
        score[1]+=1
        ballV=[1.0,0.0]
        print("P1:",score[0]," P2:",score[1])
        #Reset Inputs and Outputs
        resArrays()
        player1.pos=[0,150]
        player2.pos=[390,150]

    #Gae System
    #print(ballV[1])

    #Drawning
    screen.fill(c_off)
    player1.print()
    player2.print()
    ball.print()
    pygame.display.flip()
    clock.tick(60)


import keras
from keras import layers
from keras.datasets import imdb
from keras.preprocessing import sequence
from keras.utils import plot_model

max_features = 2000
max_len = 500

print("载入数据")
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
print("构造模型")
model = keras.models.Sequential()
model.add(layers.Embedding(max_features, 128, input_length=max_len, name='embed'))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
print("模型结构")
print(model.summary())
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
print("添加回调函数记录信息")
callbacks = [
    keras.callbacks.TensorBoard(
        log_dir='log', # 日志写入位置
        histogram_freq=1, # 每一轮都记录激活直方图
        embeddings_freq=1, # 每一轮之后记录嵌入数据
        embeddings_data=x_test[:100] # 这里需要添加这一行，原文没有
    )
]
history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.2, callbacks=callbacks)
print("使用 tensorboard --logdir=log 启动，访问 localhost:6006")
print("绘制模型结构图片")
plot_model(model, to_file='log/model.png', show_shapes=True)


from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
from django.contrib.sitemaps.views import sitemap
from django.urls import include, path, re_path

from podcasts import sitemaps, views

urlpatterns = (
    [
        path("admin/", admin.site.urls),
        path("health/", views.health, name="health"),
        path("", views.IndexView.as_view(), name="home"),
        path("vsi-podcasti/", views.AllPodcastsView.as_view(), name="all-podcasts"),
        path(
            "<slug:podcast_slug>/ep/<slug:episode_slug>/",
            views.EpisodeView.as_view(),
            name="episode",
        ),
        path(
            "<str:category_slug>-epizode/",
            views.EpisodeCategoryView.as_view(),
            name="latest-episodes-in-category",
        ),
        path("<slug:podcast_slug>/", views.PodcastView.as_view(), name="podcast"),
        path(
            "sitemap.xml",
            sitemap,
            {"sitemaps": sitemaps.sitemaps_dict},
            name="django.contrib.sitemaps.views.sitemap",
        ),
    ]
    + [re_path(r"^api/", include(views.router.urls))]
    + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
)


from django.urls import path
from rest_framework_simplejwt.views import TokenRefreshView

from .views import (
    ViewRegisterAccount,
    ViewRegisterUser, LoginView,
    ViewListUser, ViewListStatus, ProfileView, ViewRegisterNewUser
)

urlpatterns = [
    path('register/account/', ViewRegisterAccount.as_view(), name='register-account'),
    path('register/user/', ViewRegisterUser.as_view(), name='register-user'),
    path('register/profile/', ViewRegisterNewUser.as_view(), name='register-profile'),
    path('login/', LoginView.as_view(), name='token'),
    path('refresh/', TokenRefreshView.as_view(), name='token_refresh'),
    path('user_list/', ViewListUser.as_view(), name='employee-list'),
    path('status_list/', ViewListStatus.as_view(), name='status-list'),
    path('profile/', ProfileView.as_view(), name='profile'),
]


from santas_little_helpers import day, get_data, timed
from dataclasses import dataclass

today = day(2022, 18)

@dataclass
class Point:
  x: int = 0
  y: int = 0
  z: int = 0

  def __lt__(self, other):
    return self.x < other.x or self.y < other.y or self.z < other.z
  def __hash__(self):
    return 2971215073 * self.x + 433494437 * self.y + 479001599 * self.z

  @property
  def neighbours(self):
    neighbours = set()
    neighbours.add(Point(self.x+1, self.y,   self.z  ))
    neighbours.add(Point(self.x-1, self.y,   self.z  ))
    neighbours.add(Point(self.x,   self.y+1, self.z  ))
    neighbours.add(Point(self.x,   self.y-1, self.z  ))
    neighbours.add(Point(self.x,   self.y,   self.z+1))
    neighbours.add(Point(self.x,   self.y,   self.z-1))
    return neighbours

  def clear(self, others):
    return sum(1 for n in self.neighbours if n not in others)

  def covered(self, others):
    return sum(1 for n in self.neighbours if n in others)

def exterior_surface_area(lava_droplet):
  max_x = max(p.x for p in lava_droplet)
  max_y = max(p.y for p in lava_droplet)
  max_z = max(p.z for p in lava_droplet)

  minimum = Point(-1, -1, -1)
  maximum = Point(max_x + 1, max_y + 1, max_z + 1)

  to_check = [Point(0, 0, 0)]
  seen = set()
  while to_check:
    p = to_check.pop()
    if p in seen or p < minimum or p > maximum:
      continue
    yield p.covered(lava_droplet)
    seen.add(p)
    to_check.extend(p.neighbours - lava_droplet)

def main():
  inp = get_data(today, [('split', ','), ('map', int)])
  lava_droplet = set([Point(*p) for p in inp])
  print(f'{today} star 1 = {sum(c.clear(lava_droplet) for c in lava_droplet)}')
  print(f'{today} star 2 = {sum(exterior_surface_area(lava_droplet))}')

if __name__ == '__main__':
  timed(main)


# SPDX-FileCopyrightText: : 2017-2022 The PyPSA-EUR Authors, The PyPSA-Earth Authors, The PyPSA-ZA Authors
# SPDX-License-Identifier: MIT

# coding: utf-8

"""
Adds electrical generators, load and existing hydro storage units to a base network.
Relevant Settings
-----------------
.. code:: yaml
    costs:
        year:
        USD_to_ZAR:
        EUR_to_ZAR:
        marginal_cost:
        dicountrate:
        emission_prices:
        load_shedding:
    electricity:
        max_hours:
        marginal_cost:
        capital_cost:
        conventional_carriers:
        co2limit:
        extendable_carriers:
        include_renewable_capacities_from_OPSD:
        estimate_renewable_capacities_from_capacity_stats:
    load:
        scale:
        ssp:
        weather_year:
        prediction_year:
        region_load:
    renewable:
        hydro:
            carriers:
            hydro_max_hours:
            hydro_capital_cost:
    lines:
        length_factor:
.. seealso::
    Documentation of the configuration file ``config.yaml`` at :ref:`costs_cf`,
    :ref:`electricity_cf`, :ref:`load_cf`, :ref:`renewable_cf`, :ref:`lines_cf`
Inputs
------
- ``model_file.xlsx``: The database to setup different scenarios based on cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
- ``data/Eskom EAF data.xlsx``: Hydropower plant store/discharge power capacities, energy storage capacity, and average hourly inflow by country.  Not currently used!
- ``data/eskom_pu_profiles.csv``: alternative to capacities above; not currently used!
- ``data/bundle/SystemEnergy2009_22.csv`` Hourly country load profiles produced by GEGIS
- ``resources/regions_onshore.geojson``: confer :ref:`busregions`
- ``resources/gadm_shapes.geojson``: confer :ref:`shapes`
- ``data/bundle/supply_regions/{regions}.shp``: confer :ref:`powerplants`
- ``resources/profile_{}_{regions}_{resarea}.nc``: all technologies in ``config["renewables"].keys()``, confer :ref:`renewableprofiles`.
- ``networks/base_{model_file}_{regions}.nc``: confer :ref:`base`
Outputs
-------
- ``networks/elec_{model_file}_{regions}_{resarea}.nc``:
    .. image:: ../img/elec.png
            :scale: 33 %
Description
-----------
The rule :mod:`add_electricity` ties all the different data inputs from the preceding rules together into a detailed PyPSA network that is stored in ``networks/elec.nc``. It includes:
- today's transmission topology and transfer capacities (in future, optionally including lines which are under construction according to the config settings ``lines: under_construction`` and ``links: under_construction``),
- today's thermal and hydro power generation capacities (for the technologies listed in the config setting ``electricity: conventional_carriers``), and
- today's load time-series (upsampled in a top-down approach according to population and gross domestic product)
It further adds extendable ``generators`` with **zero** capacity for
- photovoltaic, onshore and AC- as well as DC-connected offshore wind installations with today's locational, hourly wind and solar capacity factors (but **no** current capacities),
- additional open- and combined-cycle gas turbines (if ``OCGT`` and/or ``CCGT`` is listed in the config setting ``electricity: extendable_carriers``)
"""

from email import generator
import logging
import os
import geopandas as gpd
import numpy as np
import pandas as pd
import powerplantmatching as pm
import pypsa
import xarray as xr
from _helpers import (configure_logging,
                    update_p_nom_max,
                    pdbcast,
                    map_generator_parameters,
                    clean_pu_profiles,
                    remove_leap_day)

from shapely.validation import make_valid
from shapely.geometry import Point
from vresutils import transfer as vtransfer
idx = pd.IndexSlice
logger = logging.getLogger(__name__)
from pypsa.descriptors import get_switchable_as_dense as get_as_dense

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning) # Comment out for debugging and development

def normed(s):
    return s / s.sum()

def calculate_annuity(n, r):
    """
    Calculate the annuity factor for an asset with lifetime n years and
    discount rate of r, e.g. annuity(20, 0.05) * 20 = 1.6
    """
    if isinstance(r, pd.Series):
        return pd.Series(1 / n, index=r.index).where(
            r == 0, r / (1.0 - 1.0 / (1.0 + r) ** n)
        )
    elif r > 0:
        return r / (1.0 - 1.0 / (1.0 + r) ** n)
    else:
        return 1 / n

def _add_missing_carriers_from_costs(n, costs, carriers):
    missing_carriers = pd.Index(carriers).difference(n.carriers.index)
    if missing_carriers.empty: return

    emissions_cols = costs.columns.to_series()\
                           .loc[lambda s: s.str.endswith('_emissions')].values
    suptechs = missing_carriers.str.split('-').str[0]
    emissions = costs.loc[suptechs, emissions_cols].fillna(0.)
    emissions.index = missing_carriers
    n.import_components_from_dataframe(emissions, 'Carrier')

def load_costs(model_file, cost_scenario, config, elec_config, config_years):
    """
    set all asset costs and other parameters
    """
    cost_data = pd.read_excel(
        model_file,
        sheet_name='costs',
        index_col=list(range(3)
        )
    ).sort_index().loc[cost_scenario]

    cost_data.drop('source',axis=1,inplace=True)

    # Interpolate for years in config file but not in cost_data excel file
    config_years_array = np.array(config_years)
    missing_year = config_years_array[~np.isin(config_years_array,cost_data.columns)]
    if len(missing_year) > 0:
        for i in missing_year:
            cost_data.insert(0,i,np.nan) # add columns of missing year to dataframe
        cost_data_tmp = cost_data.drop('unit',axis=1).sort_index(axis=1)
        cost_data_tmp = cost_data_tmp.interpolate(axis=1)
        cost_data = pd.concat([cost_data_tmp, cost_data['unit']],ignore_index=False,axis=1)

    # correct units to MW and ZAR
    cost_data.loc[cost_data.unit.str.contains("/kW")==True, config_years] *= 1e3
    cost_data.loc[cost_data.unit.str.contains("USD")==True, config_years] *= config["USD_to_ZAR"]
    cost_data.loc[cost_data.unit.str.contains("EUR")==True, config_years] *= config["EUR_to_ZAR"]

    # Convert fuel cost from R/GJ to R/MWh
    cost_data.loc[cost_data.unit.str.contains("R/GJ")==True, config_years] *= 3.6

    # Get entries where FOM is specified as % of CAPEX
    fom_perc_capex=cost_data.loc[cost_data.unit.str.contains("%/year")==True, config_years]
    fom_perc_capex=fom_perc_capex.index.get_level_values(0)

    costs = {}
    for y in config_years:
        costs[y]=cost_data.loc[idx[:, y]].unstack(level=1).fillna(
            {
                "CO2 intensity": 0,
                "FOM": 0,
                "VOM": 0,
                "discount rate": config["discountrate"],
                "heat_rate":0,
                "efficiency": 1,
                "efficiency_store": 1,
                "efficiency_dispatch": 1,
                "fuel": 0,
                "investment": 0,
                "lifetime": 25,
            }
        )

        costs[y]['efficiency_store']=costs[y]['efficiency'].pow(1./2) #if only 1 efficiency value is given assume it is round trip efficiency
        costs[y]['efficiency_dispatch']=costs[y]['efficiency'].pow(1./2)

        costs[y].loc[fom_perc_capex,'FOM']*=costs[y].loc[fom_perc_capex,"investment"]/100.0
        costs[y]["capital_cost"] = (costs[y]["investment"]*
                                    calculate_annuity(costs[y]["lifetime"], costs[y]["discount rate"])
                                    + costs[y]["FOM"])

        costs[y].at["OCGT", "fuel"] = costs[y].at["gas", "fuel"]
        costs[y].at["CCGT", "fuel"] = costs[y].at["gas", "fuel"]

        costs[y]["marginal_cost"] = costs[y]["VOM"] + costs[y]["fuel"] / costs[y]["efficiency"]

        costs[y] = costs[y].rename(columns={"CO2 intensity": "co2_emissions"})

        costs[y].at["OCGT", "co2_emissions"] = costs[y].at["gas", "co2_emissions"]
        costs[y].at["CCGT", "co2_emissions"] = costs[y].at["gas", "co2_emissions"]

        costs[y].at["solar", "capital_cost"] = 0.5 * (
            costs[y].at["solar-rooftop", "capital_cost"]
            + costs[y].at["solar-utility", "capital_cost"]
        )

        def costs_for_storage(store, link1, link2=None, max_hours=1.0):
            capital_cost = link1["capital_cost"] + max_hours * store["capital_cost"]
            if link2 is not None:
                capital_cost += link2["capital_cost"]
            return pd.Series(
                dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0)
            )

        max_hours = elec_config["max_hours"]
        costs[y].loc["battery"] = costs_for_storage(
            costs[y].loc["battery storage"],
            costs[y].loc["battery inverter"],
            max_hours=max_hours["battery"],
        )
        costs[y].loc['battery',:].fillna(costs[y].loc['battery inverter',:],inplace=True)

    for attr in ("marginal_cost", "capital_cost"):
        overwrites = config.get(attr)
        if overwrites is not None:
            overwrites = pd.Series(overwrites)
            costs[y].loc[overwrites.index, attr] = overwrites

    return costs

def add_generator_availability(n,generators,config_avail,eaf_projections):
    # Add plant availability based on actual Eskom data
    eskom_data  = pd.read_excel(
        snakemake.input.existing_generators_eaf,
        sheet_name='eskom_data',
        na_values=['-'],
        index_col=[1,0],
        parse_dates=True
    )
    snapshots = n.snapshots.get_level_values(1)
    years_full = range(n.investment_periods[0]-1,n.investment_periods[-1]+1)
    eaf_profiles = pd.DataFrame(1,index=snapshots,columns=[])

    # All existing generators in the Eskom fleet with available data
    for tech in n.generators.index[n.generators.plant_name.isin(eskom_data.index.get_level_values(0).unique())]:
        plant_name = n.generators.plant_name[tech]
        reference_data = eskom_data.loc[plant_name].loc[eskom_data.loc[plant_name].index.year.isin(config_avail['reference_years'])]
        base_eaf=(reference_data['EAF %']/100).groupby(reference_data['EAF %'].index.month).mean()
        carrier = n.generators.carrier[tech]
        for y in n.investment_periods:
            eaf_profiles.loc[str(y),tech]=base_eaf[eaf_profiles.loc[str(y)].index.month].values
            if carrier + '_fleet_EAF' in eaf_projections.index:
                eaf_profiles.loc[str(y),tech]=(base_eaf[eaf_profiles.loc[str(y)].index.month].values
                                *eaf_projections.loc[carrier+ '_fleet_EAF',y]
                                /(eskom_data.loc[carrier+'_total'].loc[eskom_data.loc[carrier+'_total'].index.year.isin(config_avail['reference_years']),'EAF %'].mean()/100))

    n.generators.loc[n.generators.plant_name.isna(),'plant_name']=n.generators.index[n.generators.plant_name.isna()]
    # New plants without existing data take best performing of Eskom fleet
    for carrier in ['coal', 'OCGT', 'CCGT', 'nuclear']:
        # 0 - Reference station, 1 - reference year, 2 - multiplier
        reference_data = eskom_data.loc[config_avail['new_unit_ref'][carrier][0]]
        reference_data = reference_data.loc[reference_data.index.year.isin(config_avail['new_unit_ref'][carrier][1])]
        base_eaf=(reference_data['EAF %']/100).groupby(reference_data['EAF %'].index.month).mean()* config_avail['new_unit_ref'][carrier][2]
        for gen_ext in n.generators[(n.generators.carrier==carrier) & (n.generators.p_nom_extendable)].index:
            eaf_profiles[gen_ext]=1
            for y in n.investment_periods:
                eaf_profiles.loc[str(y),gen_ext]=base_eaf[eaf_profiles.loc[str(y)].index.month].values

    eaf_profiles[eaf_profiles>1]=1
    eaf_profiles.index=n.snapshots
    n.generators_t.p_max_pu[eaf_profiles.columns]=eaf_profiles

def add_min_stable_levels(n, generators, config_min_stable):
    # Existing generators
    for gen in generators.index:
        if ((generators.loc[gen, "min_stable"] != 0) & (gen in n.generators.index)):
            p_min_pu = n.generators_t.p_min_pu.get(gen, n.generators.p_max_pu[gen])
            n.generators_t.p_min_pu[gen] = p_min_pu * generators.loc[gen, "min_stable"]

            p_max_pu = n.generators_t.p_max_pu.get(gen, n.generators.p_max_pu[gen])
            if isinstance(p_max_pu, (pd.DataFrame, pd.Series)):
                n.generators_t.p_max_pu[gen] = p_max_pu.where(p_max_pu >= generators.loc[gen, "min_stable"], generators.loc[gen, "min_stable"])
            else:
                n.generators_t.p_max_pu[gen] = max(p_max_pu, generators.loc[gen, "min_stable"])

    # New conventional generators take defined pu_min from config_file
    for carrier in ["coal", "OCGT", "CCGT", "nuclear","biomass"]:
        for gen_ext in n.generators[(n.generators.carrier == carrier) & (~n.generators.index.isin(generators.index))].index:
            n.generators_t.p_min_pu[gen_ext] = n.generators_t.p_max_pu[gen_ext] * config_min_stable[carrier]

    n.generators_t.p_min_pu = n.generators_t.p_min_pu.fillna(0)

 ## Attach components
# ### Load

def attach_load(n, annual_demand):
    load = pd.read_csv(snakemake.input.load,index_col=[0],parse_dates=True)

    annual_demand = annual_demand.drop('unit')*1e6

    # load = load.set_index(
    #     pd.to_datetime(load['date'] + ' ' +
    #                    load['PERIOD'].astype(str) + ':00')
    #     .rename('t'))['SYSTEMENERGY']

    demand=pd.Series(0,index=n.snapshots)

    profile_demand = normed(remove_leap_day(load.loc[str(snakemake.config['years']['reference_demand_year']),'system_energy']))

    # if isinstance(n.snapshots, pd.MultiIndex):
    for y in n.investment_periods:
            demand.loc[y]=profile_demand.values*annual_demand[y]

    n.madd("Load", n.buses.index,
           bus=n.buses.index,
           p_set=pdbcast(demand, normed(n.buses.population)))

### Generate pu profiles for other_re based on Eskom data
def generate_eskom_profiles(n,config_carriers,ref_years):
    carriers= config_carriers
    if snakemake.config["enable"]["use_excel_wind_solar"][0]:
        carriers = [ elem for elem in carriers if elem not in ['onwind','solar']]

    eskom_data = (
        pd.read_csv(
            snakemake.input.eskom_profiles,skiprows=[1],
            index_col=0,parse_dates=True
        )
        .resample('1h').mean()
    )

    eskom_data  = remove_leap_day(eskom_data)
    eskom_profiles=pd.DataFrame(0,index=n.snapshots,columns=carriers)

    for carrier in carriers:
        weather_years=ref_years[carrier]
        for i in range(0,int(np.ceil(len(n.investment_periods)/len(weather_years))-1)):
            weather_years+=weather_years

        cnt=0
        # Use the default RSA hourly data (from Eskom) and extend to multiple weather years
        for y in n.investment_periods:
            eskom_profiles.loc[y,carrier] = (eskom_data.loc[str(weather_years[cnt]),carrier]
                                        .clip(lower=0., upper=1.)).values
            cnt+=1
    return eskom_profiles

def generate_excel_wind_solar_profiles(n,ref_years):
    profiles={}
    profiles['onwind'] = pd.DataFrame(0,index=n.snapshots,columns=n.buses.index)
    profiles['solar'] = pd.DataFrame(0,index=n.snapshots,columns=n.buses.index)
    # wind and solar resources can be explicitly specified in excel format
    for carrier in ['onwind','solar']:
        raw_profiles= (pd.read_excel(snakemake.config["enable"]["use_excel_wind_solar"][1],
                                    sheet_name=carrier+'_pu',
                                    skiprows=[1],
                                    index_col=0,parse_dates=True)
                                    .resample('1h').mean()
        )
        raw_profiles = remove_leap_day(raw_profiles)

        weather_years=ref_years[carrier]
        for i in range(0,int(np.ceil(len(n.investment_periods)/len(weather_years))-1)):
            weather_years+=weather_years

        cnt=0
        # Use the default RSA hourly data (from Eskom) and extend to multiple weather years
        for y in n.investment_periods:
            profiles[carrier].loc[y,n.buses.index] = (
                raw_profiles.loc[str(weather_years[cnt]),n.buses.index]
                .clip(lower=0., upper=1.)
            ).values
            cnt+=1

    return profiles

### Set line costs

def update_transmission_costs(n, costs, length_factor=1.0, simple_hvdc_costs=False):
    for y in n.investment_periods:
        n.lines["capital_cost"] = (
            n.lines["length"] * length_factor * costs[y].at["HVAC overhead", "capital_cost"]
        )

        if n.links.empty:
            return

        dc_b = n.links.carrier == "DC"
        # If there are no "DC" links, then the 'underwater_fraction' column
        # may be missing. Therefore we have to return here.
        # TODO: Require fix
        if n.links.loc[n.links.carrier == "DC"].empty:
            return

        if simple_hvdc_costs:
            costs = (
                n.links.loc[dc_b, "length"]
                * length_factor
                * costs[y].at["HVDC overhead", "capital_cost"]
            )
        else:
            costs = (
                n.links.loc[dc_b, "length"]
                * length_factor
                * (
                    (1.0 - n.links.loc[dc_b, "underwater_fraction"])
                    * costs[y].at["HVDC overhead", "capital_cost"]
                    + n.links.loc[dc_b, "underwater_fraction"]
                    * costs[y].at["HVDC submarine", "capital_cost"]
                )
                + costs[y].at["HVDC inverter pair", "capital_cost"]
            )
        n.links.loc[dc_b, "capital_cost"] = costs

# ### Generators - TODO Update from pypa-eur
def attach_wind_and_solar(n, costs,input_profiles, model_setup, eskom_profiles):

    # Aggregate existing REIPPPP plants per region
    eskom_gens = pd.read_excel(
        snakemake.input.model_file,
        sheet_name='existing_eskom',
        na_values=['-'],
        index_col=[0,1]
    ).loc[model_setup.existing_eskom]
    eskom_gens = eskom_gens[eskom_gens['Carrier'].isin(['solar','onwind'])] # Currently only Sere wind farm

    ipp_gens = pd.read_excel(
        snakemake.input.model_file,
        sheet_name='existing_non_eskom',
        na_values=['-'],
        index_col=[0,1]
    ).loc[model_setup.existing_non_eskom]
    ipp_gens=ipp_gens[ipp_gens['Carrier'].isin(['solar','onwind'])] # add existing wind and PV IPP generators

    gens = pd.concat([eskom_gens,ipp_gens])
    gens['bus']=np.nan
    # Calculate fields where pypsa uses different conventions
    gens = map_generator_parameters(gens,n.investment_periods[0])

    # Associate every generator with the bus of the region it is in or closest to
    pos = gpd.GeoSeries([Point(o.x, o.y) for o in gens[['x', 'y']].itertuples()], index=gens.index)
    regions = gpd.read_file(
        snakemake.input.supply_regions,
        layer = snakemake.wildcards.regions
    ).to_crs(snakemake.config["crs"]["geo_crs"]).set_index('name')

    for bus, region in regions.geometry.items():
        pos_at_bus_b = pos.within(region)
        if pos_at_bus_b.any():
            gens.loc[pos_at_bus_b, "bus"] = bus
    gens.loc[gens.bus.isnull(), "bus"] = pos[gens.bus.isnull()].map(lambda p: regions.distance(p).idxmin())
    gens.loc['Sere','Grouping'] = 'REIPPPP_BW1' #add Sere wind farm to BW1 for simplification #TODO fix this to be general

    # Aggregate REIPPPP bid window generators at each bus #TODO use capacity weighted average for lifetime, costs
    for carrier in ['solar','onwind']:
        plant_data = gens.loc[gens['carrier']==carrier,['Grouping','bus','p_nom']].groupby(['Grouping','bus']).sum()
        for param in ['lifetime','capital_cost','marginal_cost']:
            plant_data[param]=gens.loc[gens['carrier']==carrier,['Grouping','bus',param]].groupby(['Grouping','bus']).mean()

        resource_carrier=pd.DataFrame(0,index=n.snapshots,columns=n.buses.index)
        if ((snakemake.config["enable"]["use_eskom_wind_solar"]==False) &
            (snakemake.config["enable"]["use_excel_wind_solar"][0]==False)):
            ds = xr.open_dataset(getattr(input_profiles, "profile_" + carrier))
            for y in n.investment_periods:
                    atlite_data = ds["profile"].transpose("time", "bus").to_pandas()
                     #TODO remove hard coding only use 1yr from atlite at present
                    #resource_carrier.loc[y] = (atlite_data.loc[str(weather_years[cnt])].clip(lower=0., upper=1.)).values
                    resource_carrier.loc[y] = atlite_data.clip(lower=0., upper=1.).values[0:8760]
        elif (snakemake.config["enable"]["use_excel_wind_solar"][0]):
            excel_wind_solar_profiles = generate_excel_wind_solar_profiles(n,
                                snakemake.config['years']['reference_weather_years'])
            resource_carrier[n.buses.index] = excel_wind_solar_profiles[carrier][n.buses.index]
        else:
            for bus in n.buses.index:
                resource_carrier[bus] = eskom_profiles[carrier].values # duplicate aggregate Eskom profile if specified

        for group in plant_data.index.levels[0]:
            n.madd("Generator", plant_data.loc[group].index, suffix=" "+group+"_"+carrier,
                bus=plant_data.loc[group].index,
                carrier=carrier,
                build_year=n.investment_periods[0],
                lifetime=plant_data.loc[group,'lifetime'],
                p_nom = plant_data.loc[group,'p_nom'],
                p_nom_extendable=False,
                marginal_cost = plant_data.loc[group,'marginal_cost'],
                p_max_pu=resource_carrier[plant_data.loc[group].index],
                p_min_pu=resource_carrier[plant_data.loc[group].index]*0.95, # for existing PPAs force to buy all energy produced
                )

    # Add new generators
        for y in n.investment_periods:
            #TODO add check here to exclude buses where p_nom_max = 0
            n.madd("Generator", n.buses.index, suffix=" "+carrier+"_"+str(y),
                bus=n.buses.index,
                carrier=carrier,
                build_year=y,
                lifetime=costs[y].at[carrier,'lifetime'],
                p_nom_extendable=True,
                #p_nom_max=ds["p_nom_max"].to_pandas(), # For multiple years a further constraint is applied in prepare_network.py
                #weight=ds["weight"].to_pandas(),
                marginal_cost=costs[y].at[carrier, 'marginal_cost'],
                capital_cost=costs[y].at[carrier, 'capital_cost'],
                efficiency=costs[y].at[carrier, 'efficiency'],
                p_max_pu=resource_carrier[n.buses.index])
    return gens
# # Generators
def attach_existing_generators(n, costs, eskom_profiles, model_setup):
    # Coal, gas, diesel, biomass, hydro, pumped storage

    # Add existing conventional generators that are active
    eskom_gens = pd.read_excel(
        snakemake.input.model_file,
        sheet_name='existing_eskom',
        na_values=['-'],
        index_col=[0,1]
    ).loc[model_setup.existing_eskom]

    eskom_gens = eskom_gens[~eskom_gens['Carrier'].isin(['solar','onwind'])]

    ipp_gens = pd.read_excel(
        snakemake.input.model_file,
        sheet_name='existing_non_eskom',
        na_values=['-'],
        index_col=[0,1]
    ).loc[model_setup.existing_non_eskom]

    ipp_gens=ipp_gens[~ipp_gens['Carrier'].isin(['solar','onwind'])] # add existing non eskom generators (excluding solar, onwind)

    gens = pd.concat([eskom_gens,ipp_gens])
    gens = map_generator_parameters(gens,n.investment_periods[0])

    # CahoraBassa will be added later, even though we don't have coordinates
    CahoraBassa  = pd.DataFrame(gens.loc["CahoraBassa"]).T
    # Drop power plants where we don't have coordinates or capacity
    gens = pd.DataFrame(gens.loc[lambda df: (df.p_nom>0.) & df.x.notnull() & df.y.notnull()])

    # Associate every generator with the bus of the region it is in or closest to
    pos = gpd.GeoSeries([Point(o.x, o.y) for o in gens[['x', 'y']].itertuples()], index=gens.index)
    regions = gpd.read_file(
        snakemake.input.supply_regions,
        layer = snakemake.wildcards.regions
    ).to_crs(snakemake.config["crs"]["geo_crs"]).set_index('name')

    for bus, region in regions.geometry.items():
        pos_at_bus_b = pos.within(region)
        if pos_at_bus_b.any():
            gens.loc[pos_at_bus_b, "bus"] = bus
    gens.loc[gens.bus.isnull(), "bus"] = pos[gens.bus.isnull()].map(lambda p: regions.distance(p).idxmin())

    if snakemake.wildcards.regions=='1-supply':
        CahoraBassa['bus'] = "RSA"
    elif snakemake.wildcards.regions=='27-supply':
        CahoraBassa['bus'] = "POLOKWANE"
    else:
        CahoraBassa['bus'] = "LIMPOPO"
    gens = pd.concat([gens,CahoraBassa])

    gen_index=gens[gens.carrier.isin(['coal','nuclear','gas','diesel','hydro','hydro-import'])].index
    n.madd("Generator", gen_index,
        bus=gens.loc[gen_index,'bus'],
        carrier=gens.loc[gen_index,'carrier'],
        build_year=n.investment_periods[0],
        lifetime=gens.loc[gen_index,'lifetime'],
        p_nom = gens.loc[gen_index,'p_nom'],
        p_nom_extendable=False,
        efficiency = gens.loc[gen_index,'efficiency'],
        ramp_limit_up = gens.loc[gen_index,'ramp_limit_up'],
        ramp_limit_down = gens.loc[gen_index,'ramp_limit_down'],
        marginal_cost=gens.loc[gen_index,'marginal_cost'],
        capital_cost=gens.loc[gen_index,'capital_cost'],
        #p_max_pu - added later under generator availability function
        )
    n.generators['plant_name'] = n.generators.index.str.split('*').str[0]

    for carrier in ['CSP','biomass']:
        n.add("Carrier", name=carrier)
        plant_data = gens.loc[gens['carrier']==carrier,['Grouping','bus','p_nom']].groupby(['Grouping','bus']).sum()
        for param in ['lifetime','efficiency','capital_cost','marginal_cost']:
            plant_data[param]=gens.loc[gens['carrier']==carrier,['Grouping','bus',param]].groupby(['Grouping','bus']).mean()

        for group in plant_data.index.levels[0]:
            # Duplicate Aggregate Eskom Data across the regions
            eskom_data = pd.concat([eskom_profiles[carrier]] * (len(plant_data.loc[group].index)), axis=1, ignore_index=True)
            eskom_data.columns = plant_data.loc[group].index
            capacity_factor = (eskom_data[plant_data.loc[group].index]).mean()[0]
            annual_cost = capacity_factor * 8760 * plant_data.loc[group,'marginal_cost']

            n.madd("Generator", plant_data.loc[group].index, suffix=" "+group+"_"+carrier,
                bus=plant_data.loc[group].index,
                carrier=carrier,
                build_year=n.investment_periods[0],
                lifetime=plant_data.loc[group,'lifetime'],
                p_nom = plant_data.loc[group,'p_nom'],
                p_nom_extendable=False,
                efficiency = plant_data.loc[group,'efficiency'],
                capital_cost=annual_cost,
                p_max_pu=eskom_data.values,
                p_min_pu=eskom_data.values*0.95, #purchase at least 95% of power under existing PPAs despite higher cost
                )

    # ## HYDRO and PHS
    # # Cohora Bassa imports to South Africa - based on Actual Eskom data from 2017-2022
    n.generators_t.p_max_pu['CahoraBassa'] = eskom_profiles['hydro-import'].values
    # Hydro power generation - based on actual Eskom data from 2017-2022
    for tech in n.generators[n.generators.carrier=='hydro'].index:
        n.generators_t.p_max_pu[tech] = eskom_profiles['hydro'].values

    for tech in n.generators[n.generators.carrier=='hydro-import'].index:
        n.generators_t.p_max_pu[tech] = eskom_profiles['hydro-import'].values

    # PHS
    phs = gens[gens.carrier=='PHS']
    n.madd('StorageUnit', phs.index, carrier='PHS',
            bus=phs['bus'],
            p_nom=phs['p_nom'],
            max_hours=phs['PHS_max_hours'],
            capital_cost=phs['capital_cost'],
            marginal_cost=phs['marginal_cost'],
            efficiency_dispatch=phs['PHS_efficiency']**(0.5),
            efficiency_store=phs['PHS_efficiency']**(0.5),
            cyclic_state_of_charge=True
            #inflow=inflow_t.loc[:, hydro.index]) #TODO add in
            )

    _add_missing_carriers_from_costs(n, costs[n.investment_periods[0]], gens.carrier.unique())

    return gens

def attach_extendable_generators(n, costs):
    elec_opts = snakemake.config['electricity']
    carriers = elec_opts['extendable_carriers']['Generator']
    if snakemake.wildcards.regions=='1-supply':
        buses = dict(zip(carriers,['RSA']*len(carriers)))
    elif snakemake.wildcards.regions=='27-supply':
        buses = elec_opts['buses']['27-supply']
    else:
        buses = elec_opts['buses']['11-supply']

    _add_missing_carriers_from_costs(n, costs[n.investment_periods[0]], carriers)

    for y in n.investment_periods:
        for carrier in carriers:
            buses_i = buses.get(carrier, n.buses.index.to_list())
            if buses_i=='RSA':
                    n.add("Generator", buses_i + " " + carrier + "_"+str(y),
                        bus=buses_i,
                        p_nom_extendable=True,
                        carrier=carrier,
                        build_year=y,
                        lifetime=costs[y].at[carrier, 'lifetime'],
                        capital_cost=costs[y].at[carrier, 'capital_cost'],
                        marginal_cost=costs[y].at[carrier, 'marginal_cost'],
                        efficiency=costs[y].at[carrier, 'efficiency'])
            else:
                n.madd("Generator", buses_i, suffix=" " + carrier + "_"+str(y),
                    bus=buses_i,
                    p_nom_extendable=True,
                    carrier=carrier,
                    build_year=y,
                    lifetime=costs[y].at[carrier, 'lifetime'],
                    capital_cost=costs[y].at[carrier, 'capital_cost'],
                    marginal_cost=costs[y].at[carrier, 'marginal_cost'],
                    efficiency=costs[y].at[carrier, 'efficiency'])

def attach_storage(n, costs):
    elec_opts = snakemake.config['electricity']
    carriers = elec_opts['extendable_carriers']['StorageUnit']
    max_hours = elec_opts['max_hours']
    buses = elec_opts['buses']

    _add_missing_carriers_from_costs(n, costs[n.investment_periods[0]], carriers)

    for y in n.investment_periods:
        for carrier in carriers:
            buses_i = buses.get(carrier, n.buses.index)
            n.madd("StorageUnit", buses_i, " " + carrier + "_" + str(y),
                bus=buses_i,
                p_nom_extendable=True,
                carrier=carrier,
                build_year=y,
                capital_cost=costs[y].at[carrier, 'capital_cost'],
                marginal_cost=costs[y].at[carrier, 'marginal_cost'],
                efficiency_store=costs[y].at[carrier, 'efficiency_store'],
                efficiency_dispatch=costs[y].at[carrier, 'efficiency_dispatch'],
                max_hours=max_hours[carrier],
                cyclic_state_of_charge=True)

def add_co2limit(n):
    n.add("GlobalConstraint", "CO2Limit",
          carrier_attribute="co2_emissions", sense="<=",
          constant=snakemake.config['electricity']['co2limit'])

# already in prepare_network
    # def add_emission_prices(n, emission_prices=None, exclude_co2=False):
    #     if emission_prices is None:
    #         emission_prices = snakemake.config['costs']['emission_prices']
    #     if exclude_co2: emission_prices.pop('co2')
    #     ep = (pd.Series(emission_prices).rename(lambda x: x+'_emissions') * n.carriers).sum(axis=1)
    #     n.generators['marginal_cost'] += n.generators.carrier.map(ep)
    #     n.storage_units['marginal_cost'] += n.storage_units.carrier.map(ep)

def add_peak_demand_hour_without_variable_feedin(n):
    new_hour = n.snapshots[-1] + pd.Timedelta(hours=1)
    n.set_snapshots(n.snapshots.append(pd.Index([new_hour])))

    # Don't value new hour for energy totals
    n.snapshot_weightings[new_hour] = 0.

    # Don't allow variable feed-in in this hour
    n.generators_t.p_max_pu.loc[new_hour] = 0.

    n.loads_t.p_set.loc[new_hour] = (
        n.loads_t.p_set.loc[n.loads_t.p_set.sum(axis=1).idxmax()]
        * (1.+snakemake.config['electricity']['SAFE_reservemargin'])
    )

def add_nice_carrier_names(n, config):
    carrier_i = n.carriers.index
    nice_names = (
        pd.Series(config["plotting"]["nice_names"])
        .reindex(carrier_i)
        .fillna(carrier_i.to_series().str.title())
    )
    n.carriers["nice_name"] = nice_names
    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
    if colors.isna().any():
        missing_i = list(colors.index[colors.isna()])
        logger.warning(
            f"tech_colors for carriers {missing_i} not defined " "in config."
        )
    n.carriers["color"] = colors

#%%
if __name__ == "__main__":
    if 'snakemake' not in globals():
        from _helpers import mock_snakemake
        snakemake = mock_snakemake(
            'add_electricity',
            **{
                'model_file':'val-LC-UNC',
                'regions':'27-supply',
                'resarea':'redz',
                'll':'copt',
                'attr':'p_nom'
            }
        )

    model_setup = (
        pd.read_excel(snakemake.input.model_file,
                                sheet_name='model_setup',
                                index_col=[0])
                                .loc[snakemake.wildcards.model_file]
                                )

    projections = (pd.read_excel(snakemake.input.model_file,
                            sheet_name='projected_parameters',
                            index_col=[0,1])
                            .loc[model_setup['projected_parameters']])

    #opts = snakemake.wildcards.opts.split('-')
    n = pypsa.Network(snakemake.input.base_network)
    costs = load_costs(
        snakemake.input.model_file,
        model_setup.costs,
        snakemake.config["costs"],
        snakemake.config["electricity"],
        n.investment_periods,
    )

    #wind_solar_profiles = xr.open_dataset(snakemake.input.wind_solar_profiles).to_dataframe()
    eskom_profiles = generate_eskom_profiles(
        n,
        snakemake.config['electricity']['renewable_carriers'],
        snakemake.config['years']['reference_weather_years']
    )

    attach_load(n, projections.loc['annual_demand',:])
    if snakemake.wildcards.regions!='1-supply':
        update_transmission_costs(n, costs)
    gens = attach_existing_generators(n, costs, eskom_profiles, model_setup)
    attach_wind_and_solar(n, costs, snakemake.input, model_setup, eskom_profiles)
    attach_extendable_generators(n, costs)
    attach_storage(n, costs)
    if snakemake.config['electricity']['generator_availability']['implement_availability']==True:
        add_generator_availability(n,
            gens[gens.Type == 'Generator'],
            snakemake.config['electricity']['generator_availability'],
            projections
        )
        add_min_stable_levels(
            n,gens[gens.Type == 'Generator'],
            snakemake.config['electricity']['min_stable_levels']
        )
    add_nice_carrier_names(n, snakemake.config)
    n.export_to_netcdf(snakemake.output[0])


#!/usr/bin/env python
''' Author  : Huy Nguyen
    Program : Quick program to download web talklet for lecture
    Start   : 08/26/2016
    End     : 08/27/2016
'''

import argparse
from bs4 import BeautifulSoup
from urllib.request import urlopen # for python2  use from urllib2 import urlopen
import os
import time # to sleep to wait for the opening of browser and load the swf file

#global variable
# hardcode dictionary that map error name to right name (a pain)
dic ={
    '1':
        {'The_Cycle_of_Life':'Cycle_of_Life',
         # 'Structural_Overview_of_Biology':'Structural_Overview_Of_Biology',
         # 'Informatics_Overview_of_Biology':'Informatics_Overview_Of_Biology',
         'Informatics_Overview_of_Biology':'Informatics_Overview',
         'Cellular_Cities':'Cellular_Citties', #the alternate root need this
         'The_Core_Machinery_of_Life':'Core_Machinery_of_Life',
         #'Drug_Binding_to_Receptor':'Drug_Binding_to_Receptor',
         #'Fibrous_Proteins_in_Disease':'Fibrous_Proteins_In_Disease',
         #'Protein_in_Water':'Protein_In_Water',
#         'See_the_Protein_alone':'See_The_Protein_Alone',
#         'See_the_Bonded_Atoms':'See_The_Bonded_Atoms',
         'Fundamental_interactions':'Fundamental_Interactions',
         'Try_to_See_the_Main_Chain_1':'Try_to_See_Mainchain_1',
         'Try_to_See_the_Main_Chain_2':'Try_to_See_Mainchain_2',
         'Try_to_See_the_Main_Chain_3':'Try_to_See_Mainchain_3',
         'Try_to_See_the_Chain_Path':'Try_to_See_Chain_Path',
         'What_is_An_Atom':'What_Is_An_Atom',
         'What_is_a_Molecule':'What_Is_A_Molecule',
         'Van_Der_Waals_interaction':'Van_Der_Waals_Interaction',
         'Electrostatics_interaction':'Electrostatics_Interaction',
         'Forces_Between_atoms':'Forces_Between_Atoms',
         'Bond_angle_Bending':'Bond_Angle_Bending',
         'Water_Energy_and_Dipole':'Water_Energy_And_Dipole',
         'Units_in_Force_Fields':'Units_In_Force_Fields',
         'Explaining_the_Dielectric_Effect':'Explaining_The_Dielectric_Effect',
         'Energy_and_Springs_1':'Energy_And_Springs_1',
         'Energy_and_Springs_2':'Energy_And_Springs_2',
         'Strength_of_Interactions':'Strength_Of_Interactions',
         'Complex_interactions':'Complex_Interactions',
         'Liquids_argon_and_Water':'Liquids_Argon_and_Water',
         'Useful_tools':'Useful_Tools',
         'See_the_Protein_Alone':'See_Protein_Alone',
         'See_the_Bonded_Atoms':'See_Bonded_Atoms'},
    '2':
        {'Amino_Acids_Gly_and_Pro':'Amino_Acids_Gly_And_Pro',
         'Amino_Acids_Leu_and_Phe':'Amino_Acids_Leu_And_Phe',
         'Amino_Acids_Glu_and_Arg':'Amino_Acids_Glu_And_Arg',
         'Amino_Acids_Val_and_Ile':'Amino_Acids_Val_And_Ile',
         'Degrees_of_Freedom._Concept_2.3':'Degrees_Of_Freedom._Concept_2.3',
         'Degrees_of_Freedom':'Degrees_Of_Freedom',
         'Backbone_Degrees_of_Freedom':'Backbone_Degrees_Of_Freedom',
         'The_Alpha_Heli':'The_Alpha_Helix_',
         'Beta_Sheet':'Beta_Sheets_'},
     '3':
         {'Cath_the_Numbers':'Cath_The_Numbers',
          'Alpha-Helix_and_Beta-Sheet':'Alpha-Helix_And_Beta-Sheet',
          'Cath_Drill_Down_to_a_Level':'Cath_Drill_Down_to_A_Level'},
     '4':
         {'Entropy_and_Free_Energy1':'Entropy_and_Free_Energy',
          'Simulated_annealing':'Simulated_Annealing',
          'Molecular_Dynamics_theory':'Molecular_Dynamics_Theory',
          'Liquids_argon_and_Water':'Liquids_Argon_and_Water',
          'Voronoi_analysis_of_Contacts':'Voronoi_Analysis_of_Contacts',
          'Simulating_alpha-Helix.__Concept_4.5':'Simulating_Alpha-Helix.__Concept_4.5',
          'Dynamics_of_the_alpha-Helix':'Dynamics_of_the_Alpha-Helix',
          'Water_allows_Hydrogen_Bonds_to_Break':'Water_Allows_Hydrogen_Bonds_to_Break',
          'Los_alamos_1943-45':'Los_alamos_1943-45',
          'Units_in_Force_Fields':'Units_In_Force_Fields',
          'Los_alamos_1943-45':'Los_Alamos_1943-45'},
     '5':
         {'Normal_Mode_theory._Concept_5.1':'Normal_Mode_Theory.__Concept_5.1',
          'Basic_theory':'Basic_Theory',
          'Potential_Energy_in_torsion_Space':'Potential_Energy_in_Torsion_Space',
          'Trypsin_inhibitor_Modes':'Trypsin_Inhibitor_Modes',
          'Unfolding_alpha_Helix.__Concept_5.3':'Unfolding_Alpha_Helix.__Concept_5.3',
          'Unfold_alpha-Helix':'Unfold_Alpha-Helix',
          'Helix_Less_Stable_in_Water':'Helix_Less_Stable_In_Water',
          'Water_allows_Hydrogen_Bonds_to_Break':'Water_Allows_Hydrogen_Bonds_to_Break',
          'What_Happens_to_aromatic_Sidechains':'What_Happens_to_Aromatic_Sidechains',
          'Why_is_Folding_So_Difficult':'Why_Is_Folding_So_Difficult',
          'Folding_at_Home_Rates':'Folding_at_Home_Rates'
         },
     '6':{},
     '7':{},
     '8':{},
     '9':{}
          }

constant = 'Lecture'
apps = 'talklets'
root ='http://csb.stanford.edu/class/public/lectures/'
alter_root = 'http://www.stanford.edu/class/sbio228/public/lectures/' # in case the other sever is down

def get_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("--lecture","-l",help="Which lecture to download (1-6)")
    args = parser.parse_args()
    return args

'''@function: given the lecture name, parse the website to retrieve the name of each talklet slide
   @input   : int (lecture number)
   @output  : list (talklets website in numeric order)
'''
def parse_web(lecture):
    if lecture =='4':
        URL = root+constant+lecture+'/'
    else:
        URL = alter_root+constant+lecture+'/'
    print (URL)
    # open the url as url using url package, then use Beautiful soup to parse the website
    soup = BeautifulSoup(urlopen(URL))
    # print (soup)
    # get the text
    text = soup.text
    my_list = text.split('\n')
    # filter out the empty items
    new_list = list(filter(None,my_list))
    # do some trick to remove not important partas well as storing information
    # information to be stored: each section name of the chapter
    # for each section, store the name of the slide. Use those 2 information
    # in order to provide a full http link and append it into a list

    # list of link to download
    links =[]
    # chapter name
    # list of section names
    sections =[] # will use the length of the section to tell whether we go through
                 # all slides of a section already (since new slide starts with 'Fundamental interactions.  Concept 1.4')
    count = 0
    while count <len(new_list):
        if new_list[count] != 'CONCEPTS IN THIS LECTURE':
            count +=1
            continue
        else:
            # get the chapter name and break here
            chapter = new_list[count+1]
            count +=2 # plus 2 so it get over the index at the chapter name
            break
    # print (new_list)
    #using chapter as a mark to do sections
    while chapter not in new_list[count]:
        sections.append(new_list[count])
        count +=1
    count +=1 # increase to get over the second time it hits the chapter name
    while count < len(new_list):

        # print (new_list[count])
        if new_list[count][:2] == (lecture+'.') or (constant+' '+lecture) in new_list[count] or chapter in new_list[count]: #basically check if it is the section name, then ignore it
            count +=1
            continue
        else: # i basically ignore those 2 slides 'SB228 Lecture 2 Lecture2', 'Lecture 2 Contents'
            if new_list[count][-11:-2] == ('Concept '+lecture):
                section_index = int(new_list[count][-1])-1
                section = sections[section_index]
                # create a link to add to links
                # if it is lecture 1 then skip this part
                chapter_link = URL + constant + lecture+'/'+ section.replace(' ','_')+'/'+apps+'/'
                if lecture =='1':
                    count +=1
                    continue
            modify = new_list[count].replace(' ','_')
            if modify in dic[lecture]:
                link = chapter_link + dic[lecture][modify]+'.swf'
            else:
                link = chapter_link + modify+'.swf'
            links.append(link)
            count +=1

    return links

'''@function: Given list of website link, download the video
   @input   : list,and lecture number
   @output  : make a dir with lecture name, download all the video into that file
'''
def down_video(links,lecture):
    my_dir = './lecture'+lecture+'/'
    os.mkdir(my_dir)
    count = 0
    short = 4
    long = 6
    if lecture == '4':
        amount = long
    else:
        amount = short
    for link in links:
        count +=1
        # open the browser to load the swf file:
        cmd1= 'open '+link
        os.system(cmd1)
        time.sleep(amount)
        cmd2 ='curl -o '+my_dir+'part'+str(count)+'.swf'+' '+link
        os.system(cmd2)
        time.sleep(3)

###############################################################################
# execute the main program
###############################################################################
if __name__ == "__main__":

    args    = get_arguments()
    lecture = args.lecture
    print (lecture)
    links= parse_web(lecture)
    down_video(links,lecture)


from django.urls import path
from listaPrecio.views import listaPrecio,listap_crear,editar

urlpatterns = [
  path('',listaPrecio,name="listaPrecio"),
  path('crear/',listap_crear,name="listap-crear"),
  path('editar/<int:id>', editar, name="editar-LP"),
]


from time import gmtime, strftime

from reportlab.lib import colors
from reportlab.lib.pagesizes import letter, inch
from reportlab.platypus import Image, Paragraph, SimpleDocTemplate, Table

from lib.reportLabStyle import getWorksheetStyleSheet
from core.problem_sets import problem_number

class ReportLabDoc():
    default_specs = {
        "qr": True,
        "append_time": True,
        "header": True,
        "font": "Courier",
        "font_size": 16,
        "columns": 2,
        "paper_size": "letter",
        "folder": "sheets",
    }

    paper_size = {
        "letter" : letter
    }
    def __init__(self, problems, specs):

        self.specs = self.default_specs
        #override the defaults with anything that has been set
        for spec in specs:
            self.specs[spec] = specs[spec]

        self.name = self.specs['name']
        self.folder = self.specs['folder']
        self.problems = problems

        self.file_name = self.get_file_name()
        self.styleSheet = getWorksheetStyleSheet()
        doc = SimpleDocTemplate(self.file_name, pagesize=self.paper_size[self.specs['paper_size']])

        # container for the 'Flowable' objects

        self.worksheet = []

        if self.specs['header']:
            self.header()
        # check specs for font and style overrides
        # check specs for header and create one if asked
        # check specs for columns and create table
        # display each problem in a cell
        self.body()
        doc.build(self.worksheet)

    def get_file_name(self):
        if self.specs['append_time']:
            cur_time = strftime("%Y-%m-%d_%H-%M-%S", gmtime())
            file_name = "{}/{}_{}.pdf".format(self.folder, self.name, cur_time)
        else:
            file_name = "{}/{}.pdf".format(self.folder, self.name)
        return file_name

    def header(self):
        company_name = Paragraph("Tiny-Robot", self.styleSheet["Heading3"])
        name = Paragraph("Name:  ______________________", self.styleSheet["Heading3"])
        subject = Paragraph("Addition", self.styleSheet["Heading3"])
        date = Paragraph("Date:  ______________________", self.styleSheet["Heading3"])
        form_code = Paragraph("Form Code: {}".format(self.specs['random_seed']), self.styleSheet["Heading3"])

        # TODO change from if statements and so far, there are 4 layouts (yes/no form code and yes/no qr code)
        has_qr = self.specs.get('qr')
        has_form_code = self.specs.get('form_code')
        if has_qr:
            qr_image = Image(self.specs['qr_file'])
            data = [[company_name, qr_image], [subject, ""], [name, ""], [date, ""]]
            t=Table(data, style=[
                            ("VALIGN", (0,3), (0,3), "TOP"),
                            ('ALIGN', (1,0), (1,3),'RIGHT'),
                            ('SPAN', (1,0), (1,3)),

            ])
        elif has_form_code:
            data = [[company_name, form_code], [subject, ""], [name, ""], [date, ""]]
            t=Table(data, style=[
                            ("VALIGN", (0,3), (0,3), "TOP"),
                            ('ALIGN', (1,0), (1,3),'RIGHT'),

            ])

        else:
            data = [[company_name, name], [subject, date]]
            t=Table(data,colWidths='*', style=[
                            ('ALIGN',(0,0), (0,0),'LEFT'),
                            ('ALIGN',(1,0), (1,0),'RIGHT'),
                            ('ALIGN',(0,1), (0,1),'LEFT'),
                            ('ALIGN',(1,1), (1,1),'RIGHT'),
            ])
        self.worksheet.append(t)

    def body(self):
        data = []
        row = []
        for i, problem in enumerate(self.problems):
            problem_num_text = problem_number(i+1)
            problem_table = problem.reportlab_display()
            #print(problem_table)
            inner_data = [[problem_num_text, problem_table]]
            # TODO find height of problem, find the minimum of pages
            # space the row Heights evenly over those pages
            # or only worry about spacing evenly if it can fit to 1 page
            inner_table = Table(inner_data, colWidths='*', rowHeights=72, style=[
                ("VALIGN", (0,0), (1,0), "TOP")
            ])
            row.append(inner_table)
            if i % self.specs['columns']:
                data.append(row)
                row = []
        if row:
            data.append(row)
        t=Table(data, colWidths='*', style=[])
        self.worksheet.append(t)