"""
Here is the "write" part of the API, to signal more data is ready.
It includes the actual webhooks sent e.g. by Gitlab, as well as
API calls to update batches and outputs.
"""
import sys
import json
import yaml
import datetime
import traceback
import subprocess
from pathlib import Path

from flask import request, jsonify
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm.attributes import flag_modified

from backend import app, repos, db_session
from ..models import Project, CiCommit, Batch, Output, TestInput
from ..models.Project import update_project

from qaboard.conventions import deserialize_config

@app.route('/api/v1/batch', methods=['POST'])
@app.route('/api/v1/batch/', methods=['POST'])
def update_batch():
  data = request.get_json()
  try:
    ci_commit = CiCommit.get_or_create(
      session=db_session,
      hexsha=request.json['git_commit_sha'],
      project_id=request.json['project'],
      data=data,
    )
  except:
    return f"404 ERROR:\n ({request.json['project']}): There is an issue with your commit id ({request.json['git_commit_sha']})", 404

  batch = ci_commit.get_or_create_batch(data['batch_label'])
  # prefix_output_dir for backward-compatibility
  batch.batch_dir_override = data.get("batch_dir", data.get("prefix_output_dir"))

  # Clients can store any metadata in each batch.
  # Currently it's used by `qa optimize` to store info on iterations
  if not batch.data:
    batch.data = {}
  batch_data = request.json.get('data', {})
  # And each batch can have changes vs its commit's config and metrics.
  # The use case is usually working locally with `qa --share` and
  # seeing updated visualizations and metrics.
  if "qaboard_config" in data and data["qaboard_config"] != ci_commit.data["qatools_config"]:
    batch.data["config"] = data["qaboard_config"]
  if "qaboard_metrics" in data and data["qaboard_metrics"] != ci_commit.data["qatools_metrics"]:
    batch.data["qatools_metrics"] = data["qaboard_metrics"]
  batch.data = {**batch.data, **batch_data}

  # Save info on each "qa batch" command in the batch, mainly to list them in logs
  command = request.json.get('command')
  if command:
    batch.data["commands"] = {**batch.data.get('commands', {}), **command}
    flag_modified(batch, "data")

  # It's a `qa optimzize` experiment
  if batch_data.get('optimization'):
    if 'best_iter' in batch_data:
      # we will save the outputs from the best iteration in the batch,
      # so first we need to remove any previous best results
      for o in batch.outputs:
        if o.output_type != 'optim_iteration':
          o.delete(soft=False)
      db_session.add(batch)
      db_session.commit()
      # Move results from the best iteration in this batch
      batch_batch_label = batch_data['last_iteration_label']
      best_batch = ci_commit.get_or_create_batch(batch_batch_label)
      for o in best_batch.outputs:
        o.output_dir_override = str(o.output_dir)
        o.batch = batch
        db_session.add(o)
      db_session.commit()

      for b in ci_commit.batches:
        if b.label.startswith(f"{data['batch_label']}|iter") and b.label != batch_data['last_iteration_label']:
          print(f'Deleting previous iteration {b.label}')
          if b.label != batch_data['last_iteration_label']:
            db_session.delete(b)

  db_session.add(batch)
  db_session.commit()
  return jsonify({"status": "OK", "id": batch.id})

@app.route('/api/v1/batch/stop', methods=['POST'])
@app.route('/api/v1/batch/stop/', methods=['POST'])
def stop_batch():
  data = request.get_json()
  try:
    batch = Batch.query.filter(Batch.id == data['id']).one()
  except:
    return f"404 ERROR:\n Not found", 404
  status = batch.stop()
  return jsonify(status), 200 if not "error" in status else 500

@app.route('/api/v1/batch/redo', methods=['POST'])
@app.route('/api/v1/batch/redo/', methods=['POST'])
def redo_batch():
  data = request.get_json()
  try:
    batch = Batch.query.filter(Batch.id == data['id']).one()
  except:
    return f"404 ERROR:\n Not found", 404
  status = batch.redo(only_deleted=data.get('only_deleted', False))
  return '{"status": "OK"}'

@app.route('/api/v1/batch/rename', methods=['POST'])
@app.route('/api/v1/batch/rename/', methods=['POST'])
def rename_batch():
  data = request.get_json()
  try:
    batch = Batch.query.filter(Batch.id == data['id']).one()
  except:
    return f"404 ERROR:\n Not found", 404
  status = batch.rename(label=data['label'], db_session=db_session)
  return '{"status": "OK"}'

@app.route('/api/v1/batch/<batch_id>', methods=['DELETE'])
@app.route('/api/v1/batch/<batch_id>/', methods=['DELETE'])
def delete_batch(batch_id):
  try:
    batch = Batch.query.filter(Batch.id == batch_id).one()
  except:
    return f"404 ERROR:\nNot found", 404
  stop_status = batch.stop()
  if "error" in stop_status:
    return jsonify(stop_status), 500
  batch.delete(session=db_session, only_failed=request.args.get('only_failed', False))
  return {"status": "OK"}

@app.route('/api/v1/commit/<path:project_id>/<commit_id>/batches', methods=['DELETE'])
@app.route('/api/v1/commit/<path:project_id>/<commit_id>/batches/', methods=['DELETE'])
@app.route('/api/v1/commit/<commit_id>/batches', methods=['DELETE'])
@app.route('/api/v1/commit/<commit_id>/batches/', methods=['DELETE'])
def delete_commit(commit_id, project_id=None):
  try:
    ci_commits = CiCommit.query.filter(CiCommit.hexsha == commit_id)
    if project_id:
      ci_commits = ci_commits.filter(CiCommit.project_id == project_id)
    ci_commits = ci_commits.all()
  except Exception as e:
    return f"404 ERROR {e}: {commit_id} in {project_id}", 404
  for ci_commit in ci_commits:
    if ci_commit.hexsha in ci_commit.project.milestone_commits:
      return f"403 ERROR: Cannot delete milestones", 403
    return "OK"
    for batch in ci_commit.batches:
      stop_status = batch.stop()
      if "error" in stop_status:
        return jsonify(stop_status), 500
      batch.delete(session=db_session)
      return {"status": "OK"}

@app.route('/api/v1/output', methods=['POST'])
@app.route('/api/v1/output/', methods=['POST'])
def new_output_webhook():
  """Updates the database when we get new results."""
  data = request.get_json()
  hexsha = data.get('commit_sha', data['git_commit_sha'])
  # We get a handle on the Commit object related to our new output
  try:
    ci_commit = CiCommit.get_or_create(
      session=db_session,
      hexsha=hexsha,
      project_id=data['project'],
      data=data,
    )
  except Exception as e:
    return jsonify({"error": f"Could not find your commit ({data['git_commit_sha']}). {e}"}), 404

  ci_commit.project.latest_output_datetime = datetime.datetime.utcnow()
  ci_commit.latest_output_datetime = datetime.datetime.utcnow()

  # We make sure the Test on which we ran exists in the database
  test_input_path = data.get('rel_input_path', data.get('input_path'))
  if not test_input_path:
    return jsonify({"error": "the input path was not provided"}), 400
  test_input = TestInput.get_or_create(
    db_session,
    path=test_input_path,
    database=data['database'],
  )

  # We save the basic information about our result
  batch = ci_commit.get_or_create_batch(data['batch_label'])
  if not batch.data:
    batch.data = {}
  batch.data.update({"type": data['job_type']})
  if data.get('input_metadata'):
    test_input.data['metadata'] = data['input_metadata']
    flag_modified(test_input, "data")

  platform = data['platform']
  # for backward-compat with old clients
  if platform == 'lsf':
    platform = 'linux'

  configurations = deserialize_config(data['configuration']) if 'configuration' in data else data['configurations']
  output = Output.get_or_create(db_session,
                                         batch=batch,
                                         platform=platform,
                                         configurations=configurations,
                                         extra_parameters=data['extra_parameters'],
                                         test_input=test_input,
                                        )
  output.output_type = data.get('input_type', '')

  output.data = data.get('data', {})
  # we can only trust CI outputs to run on the exact code from the commit
  output.data["ci"] = data['job_type'] == 'ci'
  if output.deleted:
    output.deleted = False

  # prefix_output_dir for backward-compatibility
  ci_commit.commit_dir_override = data.get('artifacts_commit', data.get('commit_ci_dir'))
  output.output_dir_override = data['output_directory']

  # We update the output's status
  output.is_running = data.get('is_running', False)
  if output.is_running:
    output.is_pending = True
  else:
    output.is_pending = data.get('is_pending', False)

  # We save the output's metrics
  if not output.is_pending:
    metrics = data.get('metrics', {})
    output.metrics = metrics
    output.is_failed = data.get('is_failed', False) or metrics.get('is_failed')

  db_session.add(ci_commit)
  db_session.add(output)
  db_session.commit()
  return jsonify(output.to_dict())

@app.route('/webhook/gitlab', methods=['GET', 'POST'])
def gitlab_webhook():
  """If Gitlab calls this endpoint every push, we get avatars and update our local copy of the repo."""
  # https://docs.gitlab.com/ce/user/project/integrations/webhooks.html
  data = json.loads(request.data)
  print(data, file=sys.stderr)
  update_project(data, db_session)
  return "{status:'OK'}"


import torch
from torch import nn
from torch.autograd import Variable

import numpy as np
from einops import rearrange, reduce, repeat

class DeformConv1D(nn.Module):
    def __init__(self, inc=1, outc=1, kernel_size=3, padding=1, bias=None):
        super(DeformConv1D, self).__init__()

        self.kernel_size = kernel_size
        self.padding = padding
        self.zero_padding = nn.ConstantPad1d(kernel_size // 2, value=0)
        self.conv_kernel = nn.Conv1d(inc, outc, kernel_size=kernel_size, stride=kernel_size, bias=bias)

    def forward(self, x, offset):
        # x - b, c, 1, 1
        # offset - b k c (k = kernel_size)

        x = rearrange(x, 'b c h w -> b c (h w)')
        x = rearrange(x, 'b c n -> b n c')

        offset = offset.float()
        dtype = offset.data.type()
        ks = self.kernel_size
        N = offset.size(1) # Kernel size (only x direction)

        assert N == ks, "Offset size is wrong!"

        if self.padding:
            x = self.zero_padding(x)

        # print('x: ', x.shape)

        # (b, N,  w) - w = c
        p = self._get_p(offset, dtype)

        # (b, w, N)
        p = p.contiguous().permute(0, 2, 1)

        # print('p: ', p.shape)

        q_left = Variable(p.data, requires_grad=False).floor()
        q_right = q_left + 1

        q_left = torch.clamp(q_left[..., :N], 0, x.size(2)-1).long()
        q_right = torch.clamp(q_right[..., :N], 0, x.size(2)-1).long()

        # print('q_left: ', q_left.shape)
        # print('q_right: ', q_right.shape)

        # (b, h, w, N)
        mask = (p[..., :N].lt(self.padding)+p[..., :N].gt(x.size(2)-1-self.padding)).type_as(p)
        mask = mask.detach()
        floor_p = p - (p - torch.floor(p))
        p = p*(1-mask) + floor_p*mask
        p = torch.clamp(p[..., :N], 0, x.size(2)-1)

        # print('pnew: ', p.shape)

        # linear kernel (b, h, w, N)
        g_left = (1 + (q_left[..., :N].type_as(p) - p[..., :N]))
        g_right = (1 - (q_right[..., :N].type_as(p) - p[..., :N]))

        # print('g_left: ', g_left.shape)
        # print('g_right: ', g_right.shape)

        # (b, c, h, w, N)
        x_q_left = self._get_x_q(x, q_left, N)
        x_q_right = self._get_x_q(x, q_right, N)

        # (b, c, h, w, N)
        x_offset = g_left.unsqueeze(dim=1) * x_q_left + \
                   g_right.unsqueeze(dim=1) * x_q_right

        # print('x_offset: ', x_offset.shape)

        x_offset = self._reshape_x_offset(x_offset, ks)
        out = self.conv_kernel(x_offset)

        return out

    def _get_p_n(self, N, dtype):
        p_n_x = np.meshgrid(range(-(self.kernel_size-1)//2, (self.kernel_size-1)//2+1))[0]
        # (N, 1)
        p_n = p_n_x.flatten()
        p_n = np.reshape(p_n, (1, N, 1))
        p_n = Variable(torch.from_numpy(p_n).type(dtype), requires_grad=False)
        return p_n

    @staticmethod
    def _get_p_0(w, N, dtype):
        p_0_x = np.meshgrid(range(1, w+1))[0]
        p_0 = p_0_x.flatten().reshape(1, 1, w).repeat(N, axis=1)
        p_0 = Variable(torch.from_numpy(p_0).type(dtype), requires_grad=False)
        return p_0

    def _get_p(self, offset, dtype):
        N, w = offset.size(1), offset.size(2)

        # (1, N, 1, 1)
        p_n = self._get_p_n(N, dtype)
        # (1, N, h, w)
        p_0 = self._get_p_0(w, N, dtype)

        # print(p_n.shape, p_0.shape)
        p = p_0.to(offset.device) + p_n.to(offset.device) + offset

        return p

    def _get_x_q(self, x, q, N):
        b, channels, kernel_size = q.size()

        x = x.contiguous()

        # (b, h, w, N)
        index = q[..., :N] # offset_x

        # (b, c, h*w*N)
        index = index.contiguous().unsqueeze(dim=1).expand(-1, 1, -1, -1).contiguous().view(b, 1, -1)

        # print('x: ', x.shape, ' index: ', index.shape, ' q: ', q.shape)
        # print(index)

        x_offset = x.gather(dim=-1, index=index).contiguous().view(b, 1, channels, N)

        return x_offset

    @staticmethod
    def _reshape_x_offset(x_offset, ks):
        b, c, w, N = x_offset.size()
        x_offset = torch.cat([x_offset[..., s:s+ks].contiguous().view(b, c, w*ks) for s in range(0, N, ks)], dim=-1)
        # print(x_offset.shape)
        x_offset = x_offset.contiguous().view(b, c, w*ks)

        return x_offset


import json
from datetime import datetime
import pika

connection = pika.BlockingConnection(
    pika.ConnectionParameters('localhost')
)

channel = connection.channel()
# 'Fanout' means that every queue receives the message from the exchange
channel.exchange_declare(exchange='logs2', exchange_type='fanout', durable=True)

print("Types of message:")
print("1 - Information message")
print("2 - Error message")
message_type = input("Input the message's type")
message_code = str(datetime.now())
message_body = input("Input the message's body")
message_json = json.dumps({"type": message_type, "code": message_code, "body": message_body})

channel.basic_publish(exchange='logs2',
                      routing_key='',
                      body=message_json
                      )
print("[{}]: {} {} ".format(message_type, message_code, message_body))
connection.close()


from script_scraper import ScriptScraper

def run_script_scraper(tv_scripts, script_letters, site_url, thread_count, download_directory):
  # Initialize the "Springfield, Springfield" screen scraper
  scraper = ScriptScraper(tv_scripts,
                          script_letters,
                          site_url,
                          thread_count,
                          download_directory
                         )

  print('Starting script scraping procedure:\n\tTV:{tv}\n\tLetters:{letters}'
          .format(
            tv=str(tv_scripts),
            letters=str(script_letters)
          )
        )
  scraper.scrape_site()
  print('Script scraping procedure completed. Check log for details.')

if __name__ == '__main__':
  # Parameters for scraping scripts from "Springfield, Springfield" site
  tv_scripts = True
  script_letters = ['0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
                    'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U',
                    'V', 'W', 'X', 'Y', 'Z']
  site_url = 'https://www.springfieldspringfield.co.uk'
  thread_count = 4
  download_directory = 'test'

  # Engage!
  run_script_scraper(tv_scripts, script_letters, site_url, thread_count, download_directory)


n, y = map(int, input().split())
flag = 0

for i in range(n + 1):
    for j in range(n - i + 1):
        if i*5000 + j *10000 + (n-i-j)*1000 == y:
            print("{} {} {}" .format(j, i, n-i-j))
            exit()

print("-1 -1 -1")


from dangdangapp.models import Booksinfo
class Caritem():
    def __init__(self,book,amount):
        self.amount=amount
        self.book=book

class Car():
    def __init__(self):
        self.save_price=0
        self.total_price=0
        self.caritems=[]

    def sums(self):
        self.total_price=0
        self.save_price=0
        for i in self.caritems:
            self.total_price+=i.book.dd_price * i.amount
            self.save_price+=(i.book.price-i.book.dd_price)*i.amount

    def add_book_toCar(self,bookid,changenum=1):   #添加购物车书的数量
        for j in self.caritems:
            if j.book.id == bookid:
                j.amount+=changenum
                self.sums()
                return
        book=Booksinfo.objects.filter(id=bookid)[0]
        self.caritems.append(Caritem(book,changenum))
        self.sums()

    def change_car_count(self,bookid,amount):  #变更购物车书的数量
        for k in self.caritems:
            if k.book.id==bookid:
                k.amount=amount
            self.sums()

    def delete_book(self,bookid):    #删除购物车的书
        for b in self.caritems:
            if b.book.id==bookid:
                self.caritems.remove(b)
        self.sums()


import requests
from bs4 import BeautifulSoup
from datetime import datetime
from celery.schedules import crontab
from celery.task import periodic_task

def get_price(stock_id, date=""):
    if not date:
        date = datetime.now.date().strftime("%Y%M%d")
    r = requests.get(url="http://www.twse.com.tw/exchangeReport/STOCK_DAY?response=json&date={}&stockNo={}".format(date, stock_id))
    response_data = r.json()['data']
    l = []
    for data in response_data:
        d = {}
        d['date'] = '-'.join([str(int(data[0].split('/')[0])+1911), data[0].split('/')[1], data[0].split('/')[2]])
        d['fk_stock_id'] = stock_id
        d['open'] = data[3]
        d['high'] = data[4]
        d['low'] = data[5]
        d['close'] = data[6]
        d['price_increase'] = data[7] if 'X' not in data[7] else 0
        d['volume'] = int(int(data[1].replace(',', ''))/1000)
        l.append(d)
    r = requests.post(url="http://localhost:8000/api/v1/price/", json=l)
    print(r.status_code)

@periodic_task(run_every=10)
def start():
    get_price(2382, date=20181102)
    return


# coding: utf-8
#

import os
import hashlib
import tornado.httpclient
from logzero import logger

from .page import BaseHandler

class StaticProxyHandler(BaseHandler):
    http_client = tornado.httpclient.AsyncHTTPClient()

    async def get(self, urlpath=None):
        # print(self.request.remote_ip)
        if urlpath is None:
            urlpath = "https://"+self.request.path.lstrip("/")
        else:
            urlpath = "https://" + urlpath

        #logger.debug("path: %s", urlpath)
        m = hashlib.md5()
        m.update(urlpath.encode())

        cache_dir = os.path.expanduser("~/.weditor/cache")
        _, ext = os.path.splitext(urlpath)
        cache_path = os.path.join(cache_dir, m.hexdigest() + ext)
        if not os.path.exists(cache_path):
            request = tornado.httpclient.HTTPRequest(
                url=urlpath,
                method="GET",
                validate_cert=False # fix certificate validate error
            )
            response = await self.http_client.fetch(request)
            if not os.path.isdir(cache_dir):
                os.makedirs(cache_dir)

            with open(cache_path, 'wb') as f:
                f.write(response.body)

        with open(cache_path, 'rb') as f:
            #logger.debug("use cache assets: %s", urlpath)
            self.write(f.read())


import os
import numpy as np
import glob

image_directory = 'Val/'
expression_lst = ['Angry','Disgust','Fear','Happy','Neutral','Sad','Surprise']
lst_file = 'val_label.lst'
fo = open(lst_file,'w')
for root,_,files in os.walk(image_directory):
	if len(files)<1 or not files[0].endswith('png'):
		continue
	expression = root[root.rfind('/')+1:]
	expression_label = expression_lst.index(expression)
	for f in files:
		fo.write(os.path.join(root,f)+' '+str(expression_label)+'\n')

fo.close()


'''Test model function in PyTorch.

Testing your deep learning model

Reference:
[1] No References
'''
import torch
from tqdm import tqdm

class test:

    def __init__(self):

        self.test_losses = []
        self.test_acc    = []

    def execute(self, net, device, testloader, criterion):

        net.eval()
        test_loss = 0
        correct = 0
        total = 0
        with torch.no_grad():
            for batch_idx, (inputs, targets) in enumerate(testloader):
                inputs, targets = inputs.to(device), targets.to(device)
                outputs = net(inputs)
                loss = criterion(outputs, targets)

                test_loss += loss.item()
                _, predicted = outputs.max(1)
                total += targets.size(0)
                correct += predicted.eq(targets).sum().item()

        test_loss /= len(testloader.dataset)
        self.test_losses.append(test_loss)

        print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format(
            test_loss, correct, len(testloader.dataset),
            100. * correct / len(testloader.dataset)))

        # Save.
        self.test_acc.append(100. * correct / len(testloader.dataset))


from __future__ import print_function

import csv
import re
from collections import defaultdict

import numpy as np
import pandas as pd

def read_data(fp):
    try:
        first = pd.read_csv(fp)
        if len(first.columns)==1 or len(first.columns) == 0 :
            first = pd.read_table(fp,index_col=0)
        if len(first.columns)==0:
            print('Unresolved delimiter.')
    except:
        first = pd.read_excel(fp)

    return first

def parse_tax(tax):
    pat = '^([d,p,c,o,f,g]):(.+)\(([\.\d]+)\)$'
    m = re.match(pat, tax)
    level, name, confidence = m.groups()
    name = name.strip('"')
    return level, name, float(confidence)

def get_otu_tax(infile, otus,c=0.8,filter=None):
    """
    to-do: allow for specification of rank levels to be featured
    :param infile:
    :param c:
    :param filter: 'g' for genus only
    :return:
    """
    otu_tax = defaultdict(list)
    with open(infile) as infh:
        reader = csv.reader(infh, delimiter='\t')
        for row in reader:
            otu_id, tax_predicted, strand, tax_assigned = row
            if otu_id in otus:
                if strand != '+':
                    print('Warning: OTU in minus strand: %s' % (otu_id,))
                tax_ranks = [parse_tax(tax) for tax in tax_predicted.split(',')]
                tax_ranks_prune = [tax_rank for tax_rank in tax_ranks if tax_rank[-1] >= c]
                for i in range(len(tax_ranks_prune)):
                    tax_key = ','.join(['%s:%s' % (it[0], it[1]) for it in tax_ranks_prune[:i+1]])
                    otu_tax[tax_key].append(otu_id)
                # append OTUs (leaves of the taxonomy tree)
                tax_key_otu = ','.join([tax_key, otu_id])
                otu_tax[tax_key_otu].append(otu_id)
    if filter:
        new_otu_tax = defaultdict(list)
        for key in otu_tax.keys():
            if key.split(',')[-1].startswith('%s:' % filter):
                new_otu_tax[key.split(',')[-1]] += otu_tax[key]
        return new_otu_tax
    return otu_tax

def get_otu_table(infile):
    """
    Receive a OTU table and return a
    :param infile: absolute file path
    :return: a tuple with two dict. first is {sample_name:{OTU:reads_count in this sample}},
                                    second is {OTU:{sample_name:otu reads count in this sample}}
    """
    samples = defaultdict(dict)
    otus = {}

    with open(infile) as infh:
        header = next(infh)
        # assert header.startswith('#')
        header = header.strip()
        header = header.lstrip('#')
        sample_ids = header.split('\t')[1:]

        for line in infh:
            line = line.strip()
            if line:
                its = line.split('\t')
                otu_id = its[0]
                assert otu_id not in otus

                counts = its[1:]
                counts = [int(count) for count in counts]
                assert len(sample_ids) == len(counts)
                obs = zip(sample_ids,counts)
                obs = [it for it in obs if it[1]>0]

                otus[otu_id] = dict(obs)
                for it in obs:
                    samples[it[0]][otu_id] = it[1]
    return samples, otus

def get_tax(infile,accept_minux=False):
    """
    Receive a sintax.txt file return a dict.
    :param infile:
    :return: A dict {OTU:a string of taxonomy} string is like "d:Bacteria(1.0000),p:"Proteobacteria"(1.0000),c:Alphaproteobacteria(1.0000),o:Caulobacterales(1.0000),f:Caulobacteraceae(1.0000),g:Brevundimonas(0.9900)"
    """
    otu_tax = {}
    with open(infile) as infh:
        for line in infh:
            line = line.strip()
            if line:
                its = line.split('\t')
                if len(its) != 4:
                    continue
                otu_id = its[0].split(';size')[0]
                tax = its[1]
                strand = its[2]
                if strand == '+':
                    tax_pass = its[3]
                    assert otu_id not in otu_tax
                    otu_tax[otu_id] = tax
                else:
                    if accept_minux:
                        otu_tax[otu_id] = tax
    return otu_tax

def get_rank(tax):
    """
    Receive a string like "d:Bacteria(1.0000),p:"Proteobacteria"(1.0000),c:Alphaproteobacteria(1.0000),o:Caulobacterales(1.0000),f:Caulobacteraceae(1.0000),g:Brevundimonas(0.9900)"
    :param tax:
    :return: a dict {tax_rank:(tax_name, confidence)} like.{'d':('Bacteria',1.0000),'p':('Proteobacteria',1.0000)......}
    """
    ranks = tax.split(',')
    info = {}
    for it in ranks:
        rank, name = it.split(':')
        name, confidence = name.split('(')
        confidence = confidence.rstrip(')')
        assert rank not in info
        info[rank] = (name, float(confidence))
    return info

def tax_anno(otu_tax_file,otu_tab_file,output_anno,tax = 'f'):
    otu_tax = get_tax(otu_tax_file,accept_minux=True)
    otu_tab = defaultdict(list)
    otu_anno = defaultdict(list)
    with open(otu_tab_file) as infh:
        header = next(infh)
        header = header.rstrip()
        for line in infh:
            line = line.strip()
            if line:
                its = line.split('\t')
                otu_id = its[0]
                otu_abd = its[1:]
                otu_abd = np.asarray([int(float(it)) for it in otu_abd])
                try:
                    rank_info = get_rank(otu_tax[otu_id])
                    rank = rank_info.get(tax)
                    if rank and rank[1] >= 0.8:
                        otu_tab[rank[0]].append(otu_abd)
                        otu_anno[rank[0]].append(otu_id)
                    else:
                        for _tax in "gfocpd":
                            rank = rank_info.get(_tax)
                            if rank and rank[1] >= 0.8:
                                otu_tab['unclassfied_%s:%s' % (_tax,rank[0])].append(otu_abd)
                                otu_anno['unclassfied_%s:%s' % (_tax,rank[0])].append(otu_id)
                                break
                        #print rank_info
                except:
                    #import pdb;pdb.set_trace()
                    print('missing %s, maybe self deleted it' % otu_id)

    outfile = output_anno
    with open(outfile, 'w') as outfh:
        outfh.write(header+'\n')
        for tax, abds in otu_tab.items():
            abds = np.asarray(abds)
            abd = np.sum(abds, axis=0)
            outs = [tax] + list(abd)
            outs = [str(it) for it in outs]
            outfh.write('\t'.join(outs)+'\n')
    return otu_anno
def get_tax_profile(otu_tab, otu_tax):
    """
    otu_tab should take a form of rows(samples) by cols (OTUs)
    :param otu_tab:
    :param otu_tax:
    :return:
    """
    tax_profile = pd.DataFrame()
    for tax_name, otus in otu_tax.items():
        tax_profile[tax_name] = otu_tab.loc[:,otus].sum(axis=1)
    return tax_profile

def norm_otu(OTU_table_with_tax,outfile):
    samples, otus = get_otu_table(OTU_table_with_tax)
    #outfile = '/home/liaoth/data2/16s/170821_170519_combined_16s/XK_analysis/otutab_raw_filtered_5k_e5_genus.anno.txt'
    with open(outfile, 'w') as outfh:
        otu_ids = list(otus.keys())
        otu_ids.sort()
        header = ['sample_id']+otu_ids

        outfh.write('\t'.join(header)+'\n')
        sample_ids = list(samples.keys())
        sample_ids.sort()

        for sample_id in sample_ids:
            outs = [sample_id]
            obs = [samples[sample_id].get(otu_id,0) for otu_id in otu_ids]
            total = sum(obs)
            obs_relative = [it*1.0/total for it in obs]
            outs = outs + obs_relative
            outs = [str(it) for it in outs]
            outfh.write('\t'.join(outs)+'\n')

def remove_colon(fasta):
    new_fasta = []
    with open(fasta) as f1:
        tmp = f1.read()
    tmp = tmp.split('\n')
    for line in tmp:
        if '>OTU' in line:
            line = line.split(';')[0]
        new_fasta.append(line)
    with open(fasta.replace('fasta','fixed.fasta'),'w') as f1:
        f1.write('\n'.join(new_fasta))


class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

def print_list(head: ListNode):
    while head:
        if head.next:
            print(head.val, end = " -> ")
        else:
            print(head.val)
        head = head.next

def create_list(linkedlist):
    head = ListNode(linkedlist[0])
    head_copy = head
    for item in linkedlist[1:]:
        node = ListNode(item)
        head.next = node
        head = head.next
    return head_copy


import models as module_model
import metrics as module_metric
import losses as module_loss
import data_loaders as module_data_loader
import utils.optim as module_optimizer
import utils.lr_scheduler as module_lr_scheduler
import argparse
import torch
import numpy as np
from data_loaders import _create_transforms
from trainer import SegmentationTrainer
from parse_config import ConfigParser

# fix random seed for reproducibility
SEED = 4444
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
np.random.seed(SEED)

def main(config):
    logger = config.get_logger('train')
    joint_transforms, image_transforms, target_transforms, val_transform = _create_transforms(config)
    test_data_loader = config.init_obj('test_data_loader', module_data_loader, image_transforms=val_transform,
                                       target_transforms=target_transforms)

    criterion = config.init_obj('supervised_loss', module_loss)
    metrics = [getattr(module_metric, x) for x in config['metrics']]
    model = config.init_obj('model', module_model)
    optimizer = config.init_obj('optimizer', module_optimizer, model.parameters())
    lr_scheduler = config.init_obj('lr_scheduler', module_lr_scheduler, optimizer)

    logger.info(model)
    if config['trainer']['name'] == 'SegmentationTrainer':
        trainer = SegmentationTrainer(model, criterion, metrics, optimizer, config, lr_scheduler)
    else:
        raise NotImplementedError("Unsupported trainer")

    trainer.setup_loader(None, None, test_data_loader)

    trainer.test(save_result=False, save_for_visual=True)

if __name__ == '__main__':
    args_parser = argparse.ArgumentParser(description='Semantic Segmentation')
    args_parser.add_argument('-c', '--config', default='config.json', type=str,
                      help='config file path, default: config.json')

    args = args_parser.parse_args()
    config = ConfigParser(args.config)
    main(config)


import sys

def reverse_word_order_linewise(textfile):
	"""
	Reverse order of words linewise and write to new file; line order remains unchanged.
	Text must be tokenized already.

	Args:
	textfile: path to textfile
	"""
	with open(textfile, 'r') as infile:
		with open('reversed-' + textfile, 'w') as outfile:
			for line in infile.readlines():
				line = line.strip().split(' ')[::-1]
				outfile.write(' '.join(line) + '\n')

def main():
	file = sys.argv[1]
	reverse_word_order_linewise(file)

if __name__ == '__main__':
	main()


import simpy
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import numpy as np
from buffer import EpBuffer
from production_system import ProductionSystem
from collections import deque
import tensorflow as tf
from tensorflow import keras
import tensorflow_probability as tfp
from multiprocessing import Manager, Queue, Process
from multiprocessing.queues import Full
import multiprocessing
import csv
import datetime
import logging
import pickle

gradient_logger = logging.getLogger(__name__)
gradient_logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(message)s")
file_handler = logging.FileHandler("./logs/gradient.log")
file_handler.setFormatter(formatter)
gradient_logger.addHandler(file_handler)

def summarize_performance(path):

    wip = np.loadtxt (f"{path}/WIP.csv", delimiter=";", unpack=False)

    wip = np.mean(wip[:,[1, 3, 5]], axis=0)
    wip_total = np.mean(wip)

    parts_produced = np.loadtxt(f"{path}/PartsProduced.csv", delimiter=";", unpack=False)
    parts_produced = parts_produced[-1, 1:]

    flow_time = np.loadtxt (f"{path}/flow_time.csv", delimiter=";", unpack=False)
    mean_flow_time = np.mean(flow_time[:, -1])
    mean_cycle_time = np.mean(flow_time[:, -2])

    part_0 = flow_time[flow_time[:, 0] == 0 ][:, 2:]
    part_1 = flow_time[flow_time[:, 0] == 1 ][:, 2:]
    part_2 = flow_time[flow_time[:, 0] == 2 ][:, 2:]

    flow_time_parts = [np.mean(part[:, -1], axis=0) for part in [part_0, part_1, part_2]]
    cycle_time_parts = [np.mean(part[:, -2], axis=0) for part in [part_0, part_1, part_2]]

    print(f"\033[0;31mAverage WIP: {wip_total}\033[0m")
    for i in range(3):
        print(f"Route_{i}: {wip[i]}")

    print(f"\033[0;31mParts Produced Total: {parts_produced[0]}\033[0m")
    for i in range(1, 4):
        print(f"Part_{i-1}: {parts_produced[i]}")

    print(f"\033[0;31mParts Cycle Time: {mean_cycle_time}\033[0m")
    for i in range (3):
        print(f"Part{i}: {cycle_time_parts[i]}")

    print(f"\033[0;31mParts Flow Time: {mean_flow_time}\033[0m")
    for i in range (3):
        print(f"Part{i}: {flow_time_parts[i]}")

class RandomPolicy():
    def __init__(self, action_range, number_actions):
        self.action_range = action_range
        self.number_actions = number_actions
        self.name = f"WIP set by a random uniform dist"
    def get_action(self, state):
        return [np.random.randint(self.action_range[0], self.action_range[1]) for i in range(self.number_actions)]

class FixedPolicy():
    def __init__ (self, *argswip,  action_range):
        """
        Input:
        argswip: n number of ints according to the number of production routes to be controled by the agent
        action_range: a list [lower limit to the action, upperlimit to the action not used at the moment
        """
        self.action_range = action_range
        self.wip = [wip for wip in argswip]
        self.name = f"WIP set to {self.wip}"

    def get_action(self, state):
        return self.wip

def build_networks (layer_sizes, activations, input, istrunk=False):
    num_layers = len(layer_sizes)
    output = keras.layers.Dense(units=layer_sizes[0], kernel_initializer='glorot_normal')(input)
    output = keras.layers.ELU(alpha=3)(output)
    for i in range(1, num_layers):
        if i == num_layers - 1 and not istrunk:
            output = keras.layers.Dense(units=layer_sizes[i], activation=activations[i], kernel_initializer='glorot_normal')(output)
        else:
            output = keras.layers.Dense(units=layer_sizes[i], kernel_initializer='glorot_normal')(output)
            output = keras.layers.ELU(alpha=3)(output)

    return output

def build_model(input, output, name):
    return keras.Model(input, output, name=name)

class PPO:
    def __init__(self,
                 agent_name,
                 input_layer_size,
                 trunk_config,
                 mu_head_config,
                 cov_head_config,
                 critic_net_config,
                 action_range,
                 summary_writer=None):
        """
        Inputs:
        input_layer_size -> int
        trunk_config -> dict {"layer_sizes": [123, 123, 123 .....]}
        mu_head_config -> dict {"layer_sizes": [123, 123, 123 .....]}
        critic_net_config -> dict {"layer_sizes": [123, 123, 123 .....]}
        action_range -> tuple (0, 100)
        """

        self.action_range = action_range
        self.input_layer_size = input_layer_size
        self.trunk_config = trunk_config
        self.mu_head_config = mu_head_config
        self.cov_head_config = cov_head_config
        self.critic_net_config = critic_net_config
        self.name = f"WIP set by PPO"
        self.action_range = action_range
        self.agent_name = agent_name
        self.summary_writer = summary_writer
        self.number_action_calls = 0

    def build_models(self):

        self.input = keras.Input(shape=(self.input_layer_size), name="state", dtype=tf.float32)
        self.trunk = build_networks(**self.trunk_config, input=self.input, istrunk=True)
        mu_head = build_networks(**self.mu_head_config, input=self.trunk)
        cov_head = build_networks(**self.cov_head_config, input=self.trunk)
        critic = build_networks(**self.critic_net_config, input=self.input)

        # Creates a model for mu cov and critic
        self.actor_mu = build_model(self.input, mu_head, "actor_mu")
        self.actor_cov = build_model(self.input, cov_head, "actor_cov")
        self.critic = build_model(self.input, critic, "critic")
        #---Start Find the variables of the actor cov model
        actor_cov_n_layers_head = len(self.cov_head_config["layer_sizes"]) * 2 - 1
        # print(f"Number cov head {actor_cov_n_layers_head}")
        actor_cov_total_n_layers = len(self.actor_cov.layers)
        # print(f"Number cov Layers: {actor_cov_total_n_layers}")
        # exit()
        cov_head_variables = []

        for i in range (actor_cov_total_n_layers - actor_cov_n_layers_head, actor_cov_total_n_layers, 1):
            for variable in self.actor_cov.get_layer(index = i).trainable_variables:
                cov_head_variables.append(variable)
        #---END Find the variables of the actor cov model
        self.cov_head_variables = cov_head_variables
        # ---START Creates a way to access the  current value of the weights of the networks
        # actor_mu and actor_cov will be identical with exception of the output layer
        self.current_parameters = {"mu": [variable.numpy() for variable in self.actor_mu.trainable_variables],
                        "cov": [variable.numpy() for variable in cov_head_variables],
                        "critic": [variable.numpy() for variable in self.critic.trainable_variables]
                        }
        # for key, variables in self.current_parameters.items():
        #     print(key)
        #     for var in variables:
        #         print(var.shape)

        # ---END Creates a way to access the  current value of the weights of the networks
        # ---START Creates a way to acess the variables of the models (used to apply the gradients)
        self.variables = {"mu": self.actor_mu.trainable_variables,
                        "cov": cov_head_variables,
                        "critic": self.critic.trainable_variables}
        # ---END Creates a way to acess the variables of the models (used to apply the gradients)

        if self.agent_name == "Global Agent":

            self.trunk_old = build_networks(**self.trunk_config, input=self.input, istrunk=True)
            mu_head_old = build_networks(**self.mu_head_config, input=self.trunk_old)
            cov_head_old = build_networks(**self.cov_head_config, input=self.trunk_old)
            self.actor_mu_old = build_model(self.input, mu_head_old, "actor_mu_old")
            self.actor_cov_old = build_model(self.input, cov_head_old, "actor_cov_old")

            #---Start Find the variables of the actor cov model
            actor_cov_n_layers_head_old = len(self.cov_head_config["layer_sizes"]) * 2  - 1
            actor_cov_total_n_layers_old = len(self.actor_cov_old.layers)
            cov_head_variables_old = []
            for i in range (actor_cov_total_n_layers_old - actor_cov_n_layers_head_old, actor_cov_total_n_layers_old, 1):
                for variable in self.actor_cov_old.get_layer(index = i).trainable_variables:
                    cov_head_variables_old.append(variable)
        #---END Find the variables of the actor cov model
            self.cov_head_variables_old = cov_head_variables_old

            self.variables_old =  {"mu": self.actor_mu_old.trainable_variables,
                            "cov": cov_head_variables_old}

            self.current_parameters_old =  {"mu": [variable.numpy() for variable in self.actor_mu_old.trainable_variables],
                        "cov": [variable.numpy() for variable in cov_head_variables_old],
                        "critic": [variable.numpy() for variable in self.critic.trainable_variables]
                        }

    def get_action(self, state):
        """
        Inputs:
        state -> numpy.ndarray

        Returns:
        action -> array([[0.433598 0.12343 1.12341234]], shape=(1, 3), dtype=float32)

        Intermediate variables:
        mu -> tf.Tensor([[0.433598 0.12343 1.12341234]], shape=(1, 3), dtype=float32)
        cov -> tf.Tensor([[0.39956307 0.1234 2.12341234]], shape=(1, 3), dtype=float32)

       """
        assert state.shape == (1, self.input_layer_size), "Check the dimensions of State"
        # ---START Generates the average and standard deviation for the wip at the given stage
        mu = self.actor_mu(state)
        cov = self.actor_cov(state)
        # print(f"Average: {mu}")
        # print(f"Std: {cov}")
        # ---END Generates the average and standard deviation for the wip at the given stage

        # Computes a noram distribution
        probability_density_func = tfp.distributions.Normal(mu, cov)
        # Samples a WIP from the distribution
        action = probability_density_func.sample()
        if self.summary_writer != None:
            with self.summary_writer.as_default():
                tf.summary.histogram("Mu dist_0" + self.agent_name, tf.slice(mu,[0, 0], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Mu dist_1" + self.agent_name, tf.slice(mu, [0, 1], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Mu dist_2" + self.agent_name, tf.slice(mu, [0, 2], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Cov dist_0" + self.agent_name, tf.slice(mu,[0, 0], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Cov dist_1" + self.agent_name, tf.slice(mu,[0, 1], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Cov dist_2" + self.agent_name, tf.slice(mu,[0, 2], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Action_0" + self.agent_name, tf.slice(action,[0, 0], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Action_1" + self.agent_name, tf.slice(action,[0, 1], [1, 1]), self.number_action_calls)
                tf.summary.histogram("Action_2" + self.agent_name, tf.slice(action,[0, 2], [1, 1]), self.number_action_calls)

               # tf.summary.histogram("Mu dist_" + self.agent_name, tf.slice(mu, [0,0], [0, 1]), self.number_action_calls)
                # tf.summary.histogram("Cov dist_" + self.agent_name, cov, self.number_action_calls)
                # tf.summary.histogram("Action_" + self.agent_name, action, self.number_action_calls)

        # print(f"Action: {action}")
        self.number_action_calls += 1
        action = action.numpy()
        action = list(action[0, : ])
        wip_cap = []
        for wip in action:
            wip_cap.append(int(max(min(wip, self.action_range[1]), self.action_range[0])))
        return wip_cap

    def get_state_value(self, state):
        state_value = self.critic(state)
        return float(state_value)

class Agent:
    def __init__(self,
                 name,
                 action_range,
                 conwip,
                 production_system_configuration,
                 ppo_networks_configuration,
                 gamma,
                 current_number_episodes,
                 total_number_episodes,
                 episode_queue,
                 run_length,
                 epsilon,
                 record_statistics,
                 gradient_steps_per_episode_actor,
                 gradient_steps_per_episode_critic,
                 gradient_clipping_actor,
                 gradient_clipping_critic,
                 parameters_queue,
                 actor_optimizer_mu,
                 actor_optimizer_cov,
                 critic_optimizer,
                 entropy,
                 number_episodes_worker,
                 log_gradient_descent,
                 n_reward_returns

                 ):

        self.name = name
        self.action_range = action_range
        self.conwip = conwip
        self.buffer1 = EpBuffer()
        self.buffer2 = EpBuffer()
        self.production_system_configuration = production_system_configuration
        self.current_number_episodes = current_number_episodes  # number of episodes already run
        self.total_number_episodes = total_number_episodes  # Total number of episodes to be run
        self.episode_queue = episode_queue
        self.run_length= run_length
        self.ppo_networks_configuration = ppo_networks_configuration
        self.gamma = gamma
        self.action_range = action_range
        self.episode_queue = episode_queue
        self.epsilon = epsilon
        self.record_statistics = record_statistics
        self.gradient_steps_per_episode_actor = gradient_steps_per_episode_actor
        self.gradient_steps_per_episode_critic = gradient_steps_per_episode_critic
        self.gradient_clipping_actor = gradient_clipping_actor
        self.gradient_clipping_critic = gradient_clipping_critic
        self.parameters_queue = parameters_queue
        self.actor_optimizer_mu = actor_optimizer_mu
        self.actor_optimizer_cov = actor_optimizer_cov
        self.critic_optimizer = critic_optimizer
        self.entropy = entropy
        self.number_episodes_worker = number_episodes_worker
        self.log_gradient_descent = log_gradient_descent
        self.n_reward_returns = n_reward_returns

    def collect_episodes_training(self,
                        number_ep,
                        files,
                        logging):

        for ep in range(number_ep):
            seed = np.random.randint(1, 1000000000)
            random_seeds = [seed + 1, seed + 2, seed + 3, seed + 4, seed + 5, seed + 6, seed + 7]
            env = simpy.Environment()

            self.production_system_1 = ProductionSystem(env=env,
                             **self.production_system_configuration,
                             ep_buffer=self.buffer1,
                             policy=self.FixedPolicy, # will be defined only after instatiation of child classes
                             use_seeds=True,
                             files=files,
                             random_seeds= random_seeds,
                             logging=logging,
                             run_length=self.run_length)

            self.production_system_2 = ProductionSystem(env=env,
                                                   **self.production_system_configuration,
                                                   ep_buffer=self.buffer2,
                                                   twin_system=self.production_system_1,
                                                   policy=self.PPO, # will be defined only after instatiation of child classes
                                                   use_seeds=True,
                                                   files=files,
                                                   random_seeds=random_seeds,
                                                   logging=logging,
                                                   run_length=self.run_length,

                                                  )

            env.run(self.run_length)
        if files:
            return (self.production_system_1.path, self.production_system_2.path)

class GlobalAgent(Agent):
    def __init__(self,
                 name,
                 action_range,
                 conwip,
                 production_system_configuration,
                 ppo_networks_configuration,
                 gamma,
                 current_number_episodes,
                 total_number_episodes,
                 episode_queue,
                 run_length,
                 epsilon,
                 record_statistics,
                 number_of_child_agents,
                 number_episodes_worker,
                 gradient_steps_per_episode_actor,
                 gradient_steps_per_episode_critic,
                 save_checkpoints,
                 gradient_clipping_actor,
                 gradient_clipping_critic,
                 parameters_queue,
                 average_reward_queue,
                 actor_optimizer_mu,
                 actor_optimizer_cov,
                 critic_optimizer,
                 entropy,
                 log_gradient_descent,
                 n_reward_returns,
                 ):

        Agent.__init__(self,
                 name=name,
                 action_range = action_range,
                 conwip = conwip,
                 production_system_configuration = production_system_configuration,
                 current_number_episodes = current_number_episodes,
                 total_number_episodes=total_number_episodes,
                 episode_queue = episode_queue,
                 ppo_networks_configuration=ppo_networks_configuration,
                 epsilon=epsilon,
                 record_statistics=record_statistics,
                 run_length=run_length,
                 gamma=gamma,
                 gradient_steps_per_episode_actor = gradient_steps_per_episode_actor,
                 gradient_steps_per_episode_critic = gradient_steps_per_episode_critic,
                 gradient_clipping_actor=gradient_clipping_actor,
                 gradient_clipping_critic=gradient_clipping_critic,
                 parameters_queue=parameters_queue,
                 actor_optimizer_mu=actor_optimizer_mu,
                 actor_optimizer_cov=actor_optimizer_cov,
                 critic_optimizer=critic_optimizer,
                 entropy=entropy,
                 number_episodes_worker = number_episodes_worker,
                 log_gradient_descent=log_gradient_descent,
                 n_reward_returns = n_reward_returns)

        self.number_of_child_agents = number_of_child_agents
        self.save_checkpoints = save_checkpoints
        self.average_reward_queue = average_reward_queue
        self.rewards = deque(maxlen=100)

    def restore_old_models(self, ppo_config):

        try:
            self.PPO.actor_mu_old.load_weights("./saved_checkpoints/actor_mu/")
            self.PPO.actor_cov_old.load_weights("./saved_checkpoints/actor_cov/")
            self.PPO.critic.load_weights("./saved_checkpoints/critic/")

            self.PPO.actor_mu.load_weights("./saved_checkpoints/actor_mu/")
            self.PPO.actor_cov.load_weights("./saved_checkpoints/actor_cov/")

            actor_cov_n_layers_head = len(ppo_config["mu_head_config"]["layer_sizes"]) * 2 - 1
            actor_cov_total_n_layers = len(self.PPO.actor_cov.layers)
            cov_head_variables = []

            for i in range (actor_cov_total_n_layers - actor_cov_n_layers_head, actor_cov_total_n_layers, 1):
                for variable in self.PPO.actor_cov.get_layer(index = i).trainable_variables:
                    cov_head_variables.append(variable)

            self.PPO.cov_head_variables = cov_head_variables

            self.PPO.current_parameters = {"mu": [variable.numpy() for variable in self.PPO.actor_mu.trainable_variables],
                           "cov": [variable.numpy() for variable in cov_head_variables],
                           "critic": [variable.numpy() for variable in self.PPO.critic.trainable_variables]
                           }

            #check if variables in fact were loaded
            # for key, model in self.PPO.current_parameters.items():
            #     for variable in model:
            #         tf.print(variable)

            cov_head_variables_old = []

            for i in range (actor_cov_total_n_layers - actor_cov_n_layers_head, actor_cov_total_n_layers, 1):
                for variable in self.PPO.actor_cov_old.get_layer(index = i).trainable_variables:
                    cov_head_variables_old.append(variable)

            self.PPO.cov_head_variables_old = cov_head_variables_old

            self.PPO.current_parameters_old = {"mu": [variable.numpy() for variable in self.PPO.actor_mu_old.trainable_variables],
                           "cov": [variable.numpy() for variable in cov_head_variables_old],
                           "critic": [variable.numpy() for variable in self.PPO.critic.trainable_variables]
                           }

            return f"Loading models for the training policy of {self.name} was succefull"
        except Exception as message:
            return (str(message) + "\n" + f"The weights of training policy for {self.name} will be randomly generated")
    def training_loop(self):
        try:
            #---START Create a summary writer
            if self.record_statistics:
                self.writer = tf.summary.create_file_writer(f"./summaries/global/{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}")
            self.current_pass = 0
            #---END of Create summary writer

            self.FixedPolicy= FixedPolicy(self.conwip, self.conwip, self.conwip, action_range=self.action_range)
            # Creates the networks for the fixed policy
            # Creates Fixed Policy

            self.PPO = PPO(**self.ppo_networks_configuration, action_range=self.action_range, agent_name=self.name, summary_writer= self.writer)
            self.PPO.build_models() #build models (Critic, Actor Networks)
            # print(f"PPO")
            # print(f"Mu")
            # self.PPO.actor_mu.summary()
            # print(f"Cov")
            # self.PPO.actor_cov.summary()
            # exit()

            print(f"1 optimization cycle corresponds to {self.number_of_child_agents * self.number_episodes_worker} episodes and {self.gradient_steps_per_episode_actor} gradient steps")

            self.number_episodes_extracted = 0
            #---START Load variable weights if self.save_checkpoints is activated
            if self.save_checkpoints:
                try:
                    with open("./saved_checkpoints/optimizers/actor/actor_optimizer_mu.pkl", "rb") as file:
                        self.actor_optimizer_mu = pickle.load(file)
                        self.actor_optimizer_mu.learning_rate.assign(0.00001)

                    with open("./saved_checkpoints/optimizers/actor/actor_optimizer_cov.pkl", "rb") as file:
                        self.actor_optimizer_cov = pickle.load(file)
                        self.actor_optimizer_cov.learning_rate.assign(0.00001)

                    with open("./saved_checkpoints/optimizers/critic/critic_optimizer.pkl", "rb") as file:
                        self.critic_optimizer = pickle.load(file)
                        # self.critic_optimizer.learning_rate.assign(0.00001)

                except Exception as e :
                    print(e)
                    print("We couldn't load the optimizer state")
                    print("The optimizers state will be reseted")

                message = self.restore_old_models(self.ppo_networks_configuration)
                print(message)

            #---END Load variable weights if self.save_checkpoints is activated
            self.number_optimization_cycles = 0
            self.number_of_gradient_descent_steps_actor = 0
            self.number_of_gradient_descent_steps_critic = 0
            # for every optimizatin cycle there will be un x episodes and y number of gradient descent steps
            while self.current_number_episodes.value < self.total_number_episodes:
                # because workers run in pararell the number of episodes
                #would change within a cycle

                for i in range(self.number_of_child_agents):
                    #Put enough parameters for all workers
                    self.parameters_queue.put(self.PPO.current_parameters_old, block=True, timeout=30)

                #---START Record the values for the weights of policy gradient NN
                if self.record_statistics:
                    with self.writer.as_default():
                        for key, parameters in self.PPO.variables.items():
                            for variable in parameters:
                                tf.summary.histogram(f"Params_{self.name}_{str(key)}_{variable.name}", variable, self.number_optimization_cycles)
                #---END Record the values for the weights of policy gradient NN

                # Collect all the episodes waiting on the queue
                for i in range(self.number_of_child_agents * self.number_episodes_worker):
                    episode = self.episode_queue.get(block=True, timeout=1000)

                    if i == 0:
                        states, actions, next_states, rewards, qsa = episode

                        if self.record_statistics:
                            episode_reward = np.sum(rewards)
                            tf.summary.scalar(f"Rewards", episode_reward, self.number_episodes_extracted)

                    else:
                        states_temp, actions_temp, next_states_temp, rewards_temp, qsa_temp = episode

                        states = np.vstack((states, states_temp))
                        actions = np.vstack((actions, actions_temp))
                        qsa = np.vstack((qsa, qsa_temp))
                        if self.record_statistics:
                            episode_reward = np.sum(rewards_temp)
                            tf.summary.scalar(f"Rewards", episode_reward, self.number_episodes_extracted)

                    self.number_episodes_extracted += 1
                #---END collect episodes available from all worker

                for j in range(self.gradient_steps_per_episode_critic):
                    #---START gradient descent for critic
                    critic_gradient, critic_loss = self.gradient_critic(states, qsa)
                    #---START Gradient Clipping critic
                    critic_gradient = [tf.clip_by_norm(value, self.gradient_clipping_critic) for value in critic_gradient]
                    #---
                    if self.record_statistics:
                        with self.writer.as_default():
                                tf.summary.scalar("Critic_Loss", critic_loss, self.number_of_gradient_descent_steps_critic)
                    #---START Record gradient to summaries
                    if self.record_statistics:
                        with self.writer.as_default():
                            for gradient, variable in zip(critic_gradient, self.PPO.variables["critic"]):
                                tf.summary.histogram(f"Gradients_{self.name}_critic_{variable.name}", gradient, self.number_of_gradient_descent_steps_critic)

                    #---END Record gradient to summaries
                    #---START Aplly Critic Gradients
                    self.critic_optimizer.apply_gradients(zip(critic_gradient, self.PPO.variables["critic"]))
                    #---
                    #---START Observe the value of given state to see convergence
                    state = np.array([4, 5, 10, 4, 3, 2, 1, 1, 1, 0.4, 0.9, 0.7, 0.32, 10, 11, 4, 10])
                    value = self.state_value(state.reshape(1, -1))
                    value = float(value.numpy()[0])

                    if self.record_statistics:
                        with self.writer.as_default():
                            tf.summary.scalar(f"State_value", value, self.number_of_gradient_descent_steps_critic)
                    self.number_of_gradient_descent_steps_critic += 1
                    #---END Observe the value of given state to see convergence
                #---END gradient descent for critic

                #---START gradient descent for actor Nets
                for i in range(self.gradient_steps_per_episode_actor):

                    #---START Record the values for the weights of policy gradient NN
                    if self.record_statistics:
                        with self.writer.as_default():
                            for key, parameters in self.PPO.variables.items():
                                for variable in parameters:
                                    tf.summary.histogram(f"Params_{self.name}_{str(key)}_{variable.name}", variable, self.number_of_gradient_descent_steps_actor)
                    #---END Record the values for the weights of policy gradient NN

                    gradients, entropy, actor_loss = self.gradient_actor(states, actions, qsa)

                    if self.record_statistics:
                        with self.writer.as_default():
                            tf.summary.scalar("Action_Loss", actor_loss, self.number_of_gradient_descent_steps_actor)

                    #---START Cliping gradients
                    for key, gradient in gradients.items():
                        gradients[key] = [tf.clip_by_value(value, -self.gradient_clipping_actor, self.gradient_clipping_actor) for value in gradient]
                    #---END Clipping Gradients

                    #---START Record gradient to summaries
                    if self.record_statistics:
                        with self.writer.as_default():
                                for key, gradient_list in gradients.items():
                                    for gradient, variable in zip(gradient_list, self.PPO.variables[key]):

                                        tf.summary.histogram(f"Gradients_{self.name}_{str(key)}_{variable.name}", gradient, self.number_of_gradient_descent_steps_actor)
                    #---END Record gradient to summaries

                    #---Start Record average entropy for episodes
                    if self.record_statistics:
                        with self.writer.as_default():
                            tf.summary.scalar(f"Entropy", entropy, self.number_of_gradient_descent_steps_actor)
                    #---End Record average entropy'for episodes

                    #---START apply gradients for actor
                    for key, value in gradients.items():
                        if key == "mu":
                            self.actor_optimizer_mu.apply_gradients(zip(value, self.PPO.variables[key]))
                        if key == "cov":
                            self.actor_optimizer_cov.apply_gradients(zip(value, self.PPO.variables[key]))
                    self.number_of_gradient_descent_steps_actor += 1
                    #---END apply gradients for actor

                    #---END gradient descent for actor Nets

                #---START update self.current_parameter with the parameters resulting from n steps of gradient descent
                self.PPO.current_parameters = {"mu": [variable.numpy() for variable in self.PPO.actor_mu.trainable_variables],
                            "cov": [variable.numpy() for variable in self.PPO.cov_head_variables],
                            "critic": [variable.numpy() for variable in self.PPO.critic.trainable_variables]
                            }
                #---END update self.current_parameter with the parameters resulting from n steps of gradient descent
                #---START Update Old policy seting theta_old = theta
                for key, value in self.PPO.current_parameters.items():
                    if key != "critic":
                        for n, variable in enumerate(self.PPO.variables_old[key]):
                            variable.assign(value[n])
                #---END Update Old policy seting theta_old = theta

                #---START updattrigger = True self.current_parameter_old with
                self.PPO.current_parameters_old = self.PPO.current_parameters

                self.number_optimization_cycles += 1

                #---START after n iterations RUN EPISODE and PRINT REWARD
                if self.number_optimization_cycles % 1 == 0:
                    rewards_volatile = []
                for i in range (1):
                    path_conwip, path_PPO = self.collect_episodes_training(1, True, False)
                    reward_fixed = self.production_system_1.sum_rewards
                    reward_PPO = self.production_system_2.sum_rewards

                    print(f"\n Ep number: {self.current_number_episodes.value}")

                    print("--------------Conwip---------------")
                    summarize_performance(path_conwip)
                    print("--------------PPO Policy---------------")
                    summarize_performance(path_PPO)
                    print(f"\033[0;31mParts that entered the system: {sum(self.production_system_2.parts_in_system)}\033[m")
                    for i in range (3):
                        print(f"Part_{i}: {self.production_system_1.parts_in_system[i]}")

                    print(f"\033[0;32mNumber Parts delayed: {self.production_system_2.parts_delayed}\033[m")
                    print(f"\033[0;32mReward {reward_PPO}\033[m")

                    # av_reward = sum(rewards_volatile) / len(rewards_volatile)
                    # max_reward = max(rewards_volatile)
                    # min_reward = min(rewards_volatile)

                    self.number_episodes_extracted += 1 # Counts the number of episodes extraceted from episode queue and not the number o episodes already run, still yet to be extracted

                    # print(f"Ep number: {self.self.current}: Average: {av_reward} -- Max: {max_reward} -- Min {min_reward} ")
                    if self.record_statistics:
                        with self.writer.as_default():
                            tf.summary.scalar(f"Rewards", reward_PPO, self.number_episodes_extracted)
                            # tf.summary.scalar(f"Max_Reward", max_reward, self.self.current_number_episodes.value)
                            # tf.summary.scalar(f"Min_Reward", min_reward, self.self.current_number_episodes.value)
                    # self.rewards.append(av_reward)
                #---END after n iterations of the loop run episode and print reward

                #---START Save weights at current iter

                if self.save_checkpoints:

                    with open("./saved_checkpoints/optimizers/actor/actor_optimizer_mu.pkl", "wb") as file:
                        pickle.dump(self.actor_optimizer_mu, file)

                    with open("./saved_checkpoints/optimizers/actor/actor_optimizer_cov.pkl", "wb") as file:
                        pickle.dump(self.actor_optimizer_cov, file)

                    with open("./saved_checkpoints/optimizers/critic/critic_optimizer.pkl", "wb") as file:
                        pickle.dump(self.critic_optimizer, file)

                    self.PPO.actor_mu.save_weights("./saved_checkpoints/actor_mu/")
                    self.PPO.actor_cov.save_weights("./saved_checkpoints/actor_cov/")
                    self.PPO.critic.save_weights("./saved_checkpoints/critic/")

            #--- END Main RL loop
            #---After all cycles run a summary of the training session and write it on a file
            with open("Running_Log.csv", "a") as file:

                writer = csv.writer(file, delimiter=",")
                writer.writerow(["Run", self.current_number_episodes.value])
            rewards_volatile = []

            for i in range (1):

                path_conwip, path_PPO = self.collect_episodes_training(1, True, False)
                reward_fixed = self.production_system_1.sum_rewards
                reward_PPO = self.production_system_2.sum_rewards

                print(f"\n Ep number: {self.current_number_episodes.value}")

                print("--------------Conwip---------------")
                summarize_performance(path_conwip)
                print("--------------PPO Policy---------------")
                summarize_performance(path_PPO)
                print(f"\033[0;31mParts that entered the system: {sum(self.production_system_1.parts_in_system)}\033[m")

                for i in range (3):

                    print(f"Part_{i}: {self.production_system_1.parts_in_system[i]}")

                print(f"\033[0;32mNumber Parts delayed: {self.production_system_2.parts_delayed}\033[m")
                print(f"\033[0;32mReward {reward_PPO}\033[m")

            print(f"Exited Global Agent")

        except KeyboardInterrupt:
                # rewards_volatile = []
            for i in range (1):
                path_conwip, path_PPO = self.collect_episodes_training(1, True, False)
                reward_fixed = self.production_system_1.sum_rewards
                reward_PPO = self.production_system_2.sum_rewards

                print(f"\n Ep number: {self.current_number_episodes.value}")

                print("--------------Conwip---------------")
                summarize_performance(path_conwip)
                print("--------------PPO Policy---------------")
                summarize_performance(path_PPO)
                print(f"\033[0;31mParts that entered the system: {sum(self.production_system_1.parts_in_system)}\033[m")
                for i in range (3):

                    print(f"Part_{i}: {self.production_system_1.parts_in_system[i]}")

                print(f"\033[0;32mNumber Parts delayed: {self.production_system_2.parts_delayed}\033[m")
                print(f"\033[0;32mReward {reward_PPO}\033[m")

            # rewards_volatile.append(reward)
            # av_reward = sum(rewards_volatile) / len(rewards_volatile)
            # max_reward = max(rewards_volatile)
            # min_reward = min(rewards_volatile)

            print("Press ctr + C one last time. Summary has be saved!")
            # self.average_reward_queue.put(sum(self.rewards) / len(self.rewards), block=True, timeout=30)

    @tf.function(input_signature=(tf.TensorSpec(shape=[None, 17]),))
    def state_value(self, state):
        value = self.PPO.critic(state)
        return value

    @tf.function(input_signature=(tf.TensorSpec(shape=[None, 17]), tf.TensorSpec(shape=[None, 3]), tf.TensorSpec(shape=[None, 1])))
    def gradient_actor(self, states, actions, Qsa):
        with tf.GradientTape(persistent=True) as tape:
            if self.log_gradient_descent:
                gradient_logger.debug(f"Cycle {self.number_optimization_cycles}")

            #---START Actor gradient calculation
            #---START Get the parameters for the Normal dist

            if self.log_gradient_descent:
                gradient_logger.debug(f"Actions")
                for value in actions:
                    gradient_logger.debug(value)

            mu = self.PPO.actor_mu(states)

            if self.log_gradient_descent:
                gradient_logger.debug(f"MU")
                for value in mu:
                    gradient_logger.debug (value)

            cov = self.PPO.actor_cov(states)

            if self.log_gradient_descent:
                gradient_logger.debug(f"Cov")
                for value in cov:
                    gradient_logger.debug(value)

            mu_old = tf.stop_gradient(self.PPO.actor_mu_old(states))
            cov_old = tf.stop_gradient(self.PPO.actor_cov_old(states))

            if self.log_gradient_descent:
                gradient_logger.debug(f"MU_old")
                for value in mu_old:
                    gradient_logger.debug (value)

            if self.log_gradient_descent:
                gradient_logger.debug(f"Cov_old")
                for value in cov_old:
                    gradient_logger.debug (value)

            #---END Get the parameters for the Normal dist
            #---START Advantage function computation and normalization
            advantage_function = Qsa - self.PPO.critic(states)
            advantage_function_mean = tf.math.reduce_mean(advantage_function)
            advantage_function_std = tf.math.reduce_std(advantage_function)
            advantage_function = tf.math.divide(advantage_function - advantage_function_mean, (advantage_function_std + 1.0e-11))
            #---END Advantage function computation and normalization
            #---START compute the Normal distributions
            self.probability_density_func = tfp.distributions.Normal(mu, cov + 1.0e-11)
            self.probability_density_func_old = tfp.distributions.Normal(mu_old, cov_old + 1.0e-11)
            #---END compute the Normal distributions
            #---Entropy
            entropy = self.probability_density_func.entropy()
            entropy_average = tf.reduce_mean(entropy)
            entropy_average = tf.stop_gradient(entropy_average)
            #---
            #---START compute the probability of the actions taken at the current episode
            probs = self.probability_density_func.prob(actions)

            if self.log_gradient_descent:
                gradient_logger.debug(f"Probs")
                for value in probs:
                    gradient_logger.debug(value)

            probs_old = tf.stop_gradient(self.probability_density_func_old.prob(actions))

            if self.log_gradient_descent:
                gradient_logger.debug(f"Probs_old")
                for value in probs_old:
                    gradient_logger.debug(value)

            #---START Ensemble Actor loss function
            self.probability_ratio = tf.math.divide(probs + 1e-11, probs_old + 1e-11)

            if self.log_gradient_descent:
                gradient_logger.debug(f"Probability ratio")
                for value in self.probability_ratio:
                    gradient_logger.debug(value)

            cpi = tf.math.multiply(self.probability_ratio, tf.stop_gradient(advantage_function))
            clip = tf.math.minimum(cpi, tf.multiply(tf.clip_by_value(self.probability_ratio, 1 - self.epsilon, 1 + self.epsilon), tf.stop_gradient(advantage_function)))
            actor_loss = -tf.reduce_mean(clip) - entropy_average * self.entropy
            gradient_logger.debug(f"Action Loss: {actor_loss}")

            #---END Ensemble Actor loss function

        #---START Compute gradients for average
        gradients_mu = tape.gradient(actor_loss, self.PPO.actor_mu.trainable_variables)
        #---
        if self.log_gradient_descent:
            gradient_logger.debug(f"Gradients Mu")
            for value in gradients_mu:
                gradient_logger.debug(value)

        #---START Compute gradients for the covariance

        gradients_cov = tape.gradient(actor_loss, self.PPO.cov_head_variables)
        # END Compute gradients for the covariance

        if self.log_gradient_descent:
            gradient_logger.debug("Gradient Cov")
            for value in gradients_cov:
                gradient_logger.debug(value)

        gradients = {"mu": gradients_mu,
                     "cov": gradients_cov,}
        #---END Actor gradient calculation

        return gradients, entropy_average, actor_loss

    @tf.function(input_signature=(tf.TensorSpec(shape=[None, 17]), tf.TensorSpec(shape=[None, 1])))
    def gradient_critic(self, states, Qsa):
        with tf.GradientTape(persistent=True) as tape:

            critic_cost = tf.reduce_sum(tf.losses.mean_squared_error(Qsa, self.PPO.critic(states)))

        gradients_critic = tape.gradient(critic_cost, self.PPO.critic.trainable_variables)

        return gradients_critic, critic_cost

class WorkerAgent(Agent):
    def __init__(self,
                 name,
                 action_range,
                 conwip,
                 production_system_configuration,
                 ppo_networks_configuration,
                 gamma,
                 current_number_episodes,
                 total_number_episodes,
                 episode_queue,
                 number_episodes_worker,
                 run_length,
                 epsilon,
                 record_statistics,
                 gradient_clipping_actor,
                 gradient_clipping_critic,
                 parameters_queue,
                 actor_optimizer_mu,
                 actor_optimizer_cov,
                 critic_optimizer,
                 entropy,
                 gradient_steps_per_episode_actor,
                 gradient_steps_per_episode_critic,
                 log_gradient_descent,
                 n_reward_returns,
                 ):
        Agent.__init__(self,
                        name=name,
                        action_range=action_range,
                        conwip=conwip,
                        production_system_configuration=production_system_configuration,
                        current_number_episodes=current_number_episodes,
                        total_number_episodes=total_number_episodes,
                        episode_queue=episode_queue,
                        run_length=run_length,
                        epsilon=epsilon,
                        gamma=gamma,
                        record_statistics=record_statistics,
                        ppo_networks_configuration=ppo_networks_configuration,
                        gradient_clipping_actor=gradient_clipping_actor,
                        gradient_clipping_critic=gradient_clipping_critic,
                        parameters_queue=parameters_queue,
                        actor_optimizer_mu=actor_optimizer_mu,
                        actor_optimizer_cov=actor_optimizer_cov,
                        critic_optimizer=critic_optimizer,
                        entropy=entropy,
                        number_episodes_worker=number_episodes_worker,
                        gradient_steps_per_episode_actor = gradient_steps_per_episode_actor,
                        gradient_steps_per_episode_critic = gradient_steps_per_episode_critic,
                        log_gradient_descent=log_gradient_descent,
                        n_reward_returns=n_reward_returns)

    def training_loop(self):
         # ---START build NETworks for critic and actor
        if self.record_statistics:
            self.writer = tf.summary.create_file_writer(f"./summaries/global/{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}_{self.name}")

        #define fixed policy
        self.FixedPolicy = FixedPolicy(self.conwip, self.conwip, self.conwip, action_range = self.action_range)
        self.PPO = PPO(**self.ppo_networks_configuration, action_range=self.action_range, agent_name=self.name,summary_writer=self.writer)
        self.PPO.build_models() #build models (Critic, Actor Networks)
        # ---

        while self.current_number_episodes.value < self.total_number_episodes:
            self.number_episodes_run_upuntil = self.current_number_episodes.value
            # ---Update variables with information coming from gradient descent
            self.update_variables()
            # ---

            # ---START Collect n episodes from this worker
            for ep in range(self.number_episodes_worker): # Run more than episode per iteration of PPO
                if self.current_number_episodes.value < self.total_number_episodes:
                    self.collect_episodes_training(1, False, False)

                    states, actions, next_states, rewards, qsa = self.buffer2.unroll_memory(self.gamma, self.n_reward_returns, self.PPO)
                    rollout = (states, actions, next_states, rewards, qsa)
                    try:
                        self.episode_queue.put(rollout, block=False)
                    except Full:
                        print("Queue Was full")
                        break

                    self.current_number_episodes.value += 1
                else: break

            #---END Collect n episodes from this worker

        print(f"Exited {self.name}")

    def update_variables(self):
         #---Get current parameters from queue(put in by the global agent)
        try:
            self.new_params = self.parameters_queue.get(block=True)
        except Exception as e:
            print(e)

        # for key, value in self.new_params.items():
        #     print(key)
        #     for n, variable in enumerate(self.PPO.variables[key]):
        #         print(f"Variable to be passed {value[n].shape}")
        #         print(f"Variable to be updated{variable}")

        # exit()

        #---
        #---START assign the variables of the worker with the variable values from global
        for key, value in self.new_params.items():
            for n, variable in enumerate(self.PPO.variables[key]):
                variable.assign(value[n])
        #END assign the variables of the worker with the variable values from global

        #---START update variable current_parameters to reflect the information provided by global
        self.PPO.current_parameters = {"mu": [variable.numpy() for variable in self.PPO.actor_mu.trainable_variables],
                        "cov": [variable.numpy() for variable in self.PPO.cov_head_variables],
                        "critic": [variable.numpy() for variable in self.PPO.critic.trainable_variables]

                        }
        #---END update variable current_parameters to reflect the information provided by global

ppo_networks_configuration = {"trunk_config": {"layer_sizes": [100, 80, 70],
                                      "activations": ["elu",   "elu", "elu"]},

                     "mu_head_config": {"layer_sizes": [60, 50, 3],
                                        "activations": ["elu", "elu", "elu"]},
                     "cov_head_config": {"layer_sizes": [60, 50, 3],
                                        "activations": ["elu","elu", "relu"]},
                     "critic_net_config": {"layer_sizes": [100, 100, 100, 80, 40, 1],
                                            "activations": ["elu", "elu","elu", "elu", "elu", "linear"]},
                     "input_layer_size": 17
                       }

hyperparameters = {"ppo_networks_configuration" : ppo_networks_configuration,
                   "actor_optimizer_mu": tf.keras.optimizers.SGD(learning_rate=0.00001, momentum=0.9),
                   "actor_optimizer_cov": tf.keras.optimizers.SGD(learning_rate=0.00001, momentum=0.9),
                    "critic_optimizer": tf.keras.optimizers.SGD(learning_rate=0.0001, momentum=0.9),
                    "entropy": 1,
                    "gamma":0.999,
                    "gradient_clipping_actor": 1.0,
                    "gradient_clipping_critic": 1.0,
                    "gradient_steps_per_episode_critic": 5,
                    "gradient_steps_per_episode_actor": 5,
                    "epsilon": 0.1,
                    "number_episodes_worker": 10,
                    "n_reward_returns": 5
                    }
agent_config = {
    "action_range": (0, 500),
    "total_number_episodes" : 10000,
    "conwip": 500,
    "run_length": 3000

}

parts = {
    "0": {"machine_0": [0, 10], "machine_1": [0, 10], "demand": [5, 5]},
    "1": {"machine_0": [0, 5], "machine_2": [0, 20], "machine_1": [0, 20], "demand": [10, 10]},
    "2": {"machine_2": [0, 10], "machine_3": [0, 30], "demand": [15, 15]}}

production_system_config = {
    "decision_epoch_interval": 30,
    "track_state_interval": 5,
    "beta_state_weighted_average": 0.9,
    "parts_info": parts,
    "number_routes": 3,
    "number_workstations": 4,
    "number_of_different_parts": 3,
}
if __name__ == "__main__":

    multiprocessing.set_start_method('spawn')
    number_of_workers = 4

    params_queue = Manager().Queue(number_of_workers)
    current_number_episodes = Manager().Value("i", 0)
    episode_queue = Manager().Queue(number_of_workers*hyperparameters["number_episodes_worker"])
    average_reward_queue = Queue(1)
    global_agent = GlobalAgent(**hyperparameters,
                               **agent_config,
                               production_system_configuration = production_system_config,
                               name="Global Agent",
                               record_statistics=True,
                               average_reward_queue=average_reward_queue,
                               number_of_child_agents=number_of_workers,
                               save_checkpoints=True,
                               episode_queue=episode_queue,
                               current_number_episodes=current_number_episodes,
                               parameters_queue=params_queue,
                               log_gradient_descent=False)
    workers = []
    for _ in range(number_of_workers):
        print("worker created")
        myWorker = WorkerAgent(**hyperparameters,
                        **agent_config,
                        production_system_configuration=production_system_config,
                        name=f"Worker_{_}",
                        record_statistics=True,
                        episode_queue=episode_queue,
                        current_number_episodes=current_number_episodes,
                        parameters_queue=params_queue,
                        log_gradient_descent=False)

        workers.append(myWorker)

    processes = []
    p1 = Process(target=global_agent.training_loop)
    processes.append(p1)
    p1.start()
    for worker in workers:

        p = Process(target=worker.training_loop)
        processes.append(p)
        p.start()

    for p in processes:
        p.join()
    print(average_reward_queue.get())
    print("Simulation Run")


import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.ensemble import RandomForestClassifier
import statistics as stat

# Use pandas functions to preprocess the data
def preprocess_data(train_data_path, test_data_path):

    # extract data
    train_data = pd.read_csv(train_data_path)
    train_labels = train_data['Survived'].values
    test_data = pd.read_csv(test_data_path)

    # If data is missing replace with unknown flag -1 or 'UNK'
    train_data = train_data.fillna(-1)
    test_data = test_data.fillna(-1)
    train_data['Cabin'] = train_data['Cabin'].replace(-1, 'UNK')
    test_data['Cabin'] = test_data['Cabin'].replace(-1, 'UNK')

    # define mappings of categorical features to numerical values
    mf_mapping = {-1: -1, 'male': 0, 'female': 1}
    embarked_mapping = {-1:-1, 'S': 0, 'C': 1, 'Q': 2}
    cabin_mapping = create_cabin_mapping(train_data['Cabin'])

    # apply the mappings to their respective columns
    train_data['Sex'] = train_data['Sex'].map(mf_mapping)
    test_data['Sex'] = test_data['Sex'].map(mf_mapping)
    train_data['Cabin'] = train_data['Cabin'].apply(cabin_mapping_function, args=(cabin_mapping,))
    test_data['Cabin'] = test_data['Cabin'].apply(cabin_mapping_function, args=(cabin_mapping,))
    train_data['Embarked'] = train_data['Embarked'].map(embarked_mapping)
    test_data['Embarked'] = test_data['Embarked'].map(embarked_mapping)
    train_data['Ticket'] = train_data['Ticket'].apply(ticket_mapping_function)
    test_data['Ticket'] = test_data['Ticket'].apply(ticket_mapping_function)

    # drop uninformative columns
    train_data.drop(columns=['PassengerId', 'Name'])
    test_data.drop(columns=['PassengerId', 'Name'])

    # Save dataframes as pickle files
    train_data.to_pickle("train_data.pickle")
    test_data.to_pickle("test_data.pickle")
    data = {'train': train_data, 'test': test_data}
    return data

# If the ticket contains a prefix, assign a 1 else assign 0
def ticket_mapping_function(val):
    split_val = val.split(' ')
    if len(split_val) > 1:
        return 1
    else:
        return 0

# Create a dictionary that captures the first letter of the cabin number
def create_cabin_mapping(cabin_col):
    unique_vals = np.unique(cabin_col.values).tolist()
    cabin_mapping = {'U': 0}
    i = 1
    for val in unique_vals:
        if type(val) == str:
            cabin_char = val[0]
            if cabin_char not in cabin_mapping:
                cabin_mapping[cabin_char] = i
                i += 1
    return cabin_mapping

# Function to maps string cabin numbers to the cabin id:
def cabin_mapping_function(val, cabin_mapping):
    if type(val) == str:
        cabin_char = val[0]
        if cabin_char in cabin_mapping:
            return cabin_mapping[cabin_char]
    else:
        return -1

if __name__ == '__main__':
    TRAIN_DATA_PATH = 'train.csv'
    TEST_DATA_PATH = 'test.csv'
    data = preprocess_data(TRAIN_DATA_PATH, TEST_DATA_PATH)
    train_data = data['train']
    test_data = data['test']

    #PassengerId,Survived,Pclass,Name,Sex,Age,SibSp,Parch,Ticket,Fare,Cabin,Embarked
    train = pd.read_csv(TRAIN_DATA_PATH)

    print("Summary values for all passengers: ")
    print("Most common class:", stat.mode(train["Pclass"]))
    print("Most common sex:", stat.mode(train["Sex"]))
    print("Average age:", np.mean(train["Age"]))
    print("Most common SibSp:", stat.mode(train["SibSp"]))
    print("Most common Parch:", stat.mode(train["Parch"]))
    print("Average fare:", np.mean(train["Fare"]))
    print("Most common embarked:", stat.mode(train["Embarked"]))
    print()

    survivors = train.loc[train.Survived == 1]
    print("Summary values for survivors: ")
    print("Most common class:", stat.mode(survivors["Pclass"]))
    print("Most common sex:", stat.mode(survivors["Sex"]))
    print("Average age:", np.mean(survivors["Age"]))
    print("Most common SibSp:", stat.mode(survivors["SibSp"]))
    print("Most common Parch:", stat.mode(survivors["Parch"]))
    print("Average fare:", np.mean(survivors["Fare"]))
    print("Most common embarked:", stat.mode(survivors["Embarked"]))
    print()

    plt.subplot(2, 4, 1)

    women = train_data.loc[train_data.Sex == 1]["Survived"]
    rate_women = sum(women)/len(women)
    print("% of women who survived:", rate_women)

    men = train_data.loc[train_data.Sex == 0]["Survived"]
    rate_men = sum(men)/len(men)
    print("% of women who survived:", rate_men)

    plt.bar(["Female", "Male"], [rate_women, rate_men])
    plt.xlabel('Sex')
    plt.ylabel('Survival Percentage')

    class1 = train_data.loc[train_data.Pclass == 1]["Survived"]
    rate_class1 = sum(class1)/len(class1)
    print("% of class 1 who survived:", rate_class1)

    class2 = train_data.loc[train_data.Pclass == 2]["Survived"]
    rate_class2 = sum(class2)/len(class2)
    print("% of class 2 who survived:", rate_class2)

    class3 = train_data.loc[train_data.Pclass == 3]["Survived"]
    rate_class3 = sum(class3)/len(class3)
    print("% of class 3 who survived:", rate_class3)

    plt.subplot(2, 4, 2)
    plt.bar(["1st class", "2nd class", "3rd class"], [rate_class1, rate_class2, rate_class3])
    plt.xlabel('Passenger Class')

    binSize = 10
    ageBins = []
    rate_ages = []
    #print(len(train_data.loc[train_data.Age == 22]["Survived"]))
    #print(sum(train_data.loc[train_data.Age == 22]["Survived"]))
    prevLen = 0
    prevSum = 0
    for i in range(0,int(80/binSize)):
        ageBin = train_data.loc[train_data.Age <= ((i+1)*binSize)]["Survived"]
        print(sum(ageBin)-prevSum, len(ageBin)-prevLen)
        if(len(ageBin)-prevLen == 0):
            rate_ages.append(0)
        else:
            rate_ages.append((sum(ageBin)-prevSum)/(len(ageBin)-prevLen))
        prevLen = len(ageBin)
        prevSum = sum(ageBin)
        #print(prevLen, " ", prevSum)
    ages = ["10", "20", "30", "40", "50", "60", "70", "80"]
    plt.subplot(2, 4, 3)
    print(rate_ages)
    plt.bar(ages, rate_ages, width = -.9, align = 'edge')
    plt.xlabel('Age range (years)')

    binSize = 20
    numBins = int(max(train_data['Fare'])/binSize)+1
    fareBins = np.zeros(numBins)
    fareBinsSurvived = np.zeros(numBins)
    for i in range(len(train_data['Fare'])):
        bin = int(train_data['Fare'][i]/binSize)
        fareBins[bin] += 1
        fareBinsSurvived[bin] += train_data['Survived'][i]

    #print(fareBinsSurvived/fareBins)
    x = []
    for i in range(numBins):
        x.append((1+i)*binSize)

    #x = [0:binSize:numBins*binSize]
    y = fareBinsSurvived/fareBins
    y = y.tolist()
    for i in range(len(y)):
        if np.isnan(y[i]):
            y[i] = 0
    #print(x)

    plt.subplot(2, 4, 4)
    plt.bar(x, y, width=binSize*.9)
    plt.xlabel('Fare ($)')

    #print(train_data.loc[train_data.SibSp == 5])
    rate_sibsp = []
    for i in range (0,6):
        rate_sibsp.append(sum(train_data.loc[train_data.SibSp == i]["Survived"])/len(train_data.loc[train_data.SibSp == i]["Survived"]))
    x_sibsp = [0,1,2,3,4,5]
    plt.subplot(2, 4, 5)
    plt.bar(x_sibsp, rate_sibsp)
    plt.xlabel('# siblings/spouses aboard')
    plt.ylabel('Survival Percentage')

    rate_parch = []
    for i in range (0,7):
        rate_parch.append(sum(train_data.loc[train_data.Parch == i]["Survived"])/len(train_data.loc[train_data.Parch == i]["Survived"]))
    x_parch = [0,1,2,3,4,5,6]
    plt.subplot(2, 4, 6)
    plt.bar(x_parch, rate_parch)
    plt.xlabel('# parents/children aboard')

    rate_embarked = []
    x_embarked = ['C', 'Q', 'S']
    for i in x_embarked:
        print(i)
        rate_embarked.append(sum(train.loc[train.Embarked == i]["Survived"])/len(train.loc[train.Embarked == i]["Survived"]))
    plt.subplot(2, 4, 7)
    plt.bar(x_embarked, rate_embarked)
    plt.xlabel('Port of departure')

#     ss1 = train_data.loc[train_data.Pclass == 1]["Survived"]
#     rate_class1 = sum(class1)/len(class1)
#     print("% of class 1 who survived:", rate_class1)
#
#     class2 = train_data.loc[train_data.Pclass == 2]["Survived"]
#     rate_class2 = sum(class2)/len(class2)
#     print("% of class 2 who survived:", rate_class2)
#
#     class3 = train_data.loc[train_data.Pclass == 3]["Survived"]
#     rate_class3 = sum(class3)/len(class3)
#     print("% of class 3 who survived:", rate_class3)
#

    plt.show()
    #plt.plot()

    y = train_data["Survived"]

    #PassengerId,Survived,Pclass,Name,Sex,Age,SibSp,Parch,Ticket,Fare,Cabin,Embarked
    features = ["Sex", "Age", "Pclass", "Embarked", "SibSp", "Parch"]
    X = pd.get_dummies(train_data[features])
    X_test = pd.get_dummies(test_data[features])

    model = RandomForestClassifier(n_estimators=100, max_depth=20, random_state=1)
    model.fit(X, y)
    predictions = model.predict(X_test)

    print(model.feature_importances_)
    print(model.decision_path(X_test))

    from sklearn.tree import export_graphviz
    estimator = model.estimators_[5]
    # Export as dot file
    export_graphviz(estimator, out_file='tree.dot',
                    feature_names = ["Sex", "Age", "Pclass", "Embarked", "SibSp", "Parch"],
                    class_names = "Survived",
                    rounded = True, proportion = False,
                    precision = 2, filled = True)

    # Convert to png using system command (requires Graphviz)
    from subprocess import call
    #process = subprocess.Popen(command, stdout=tempFile, shell=True)
    #call(['dot', '-Tpng', 'tree.dot', '-o', 'tree.png', '-Gdpi=600'])
    #set path=%path%;C:\Anaconda3\graphviz-2.38\release\bin
    #dot -Tpng tree.dot -o tree.png

    output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
    output.to_csv('my_submission.csv', index=False)
    #score = model.score(X,y)    # test score, since gt not provided for test data
    #print("Accuracy = ", score)

# Preprocessing slide (bullets + screenshot) - R
# Features vs. Survival (scatterplot/histogram) - K
# PCA?
# Logistic Regression (confusion matrix, feature weights) - Y
# Random Forest - K
# Neural Net? (confusion matrix, overfitting plot) - R
# Conclusions - Y


import random
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

x=0
def wichtelmail(Wichtel , Wichtelemail , empfaenger):
    senderEmail = "defnitivnichtderweihnachtsmann@gmail.com"

    empfangsEmail = Wichtelemail
    msg = MIMEMultipart()
    msg['From'] = senderEmail
    msg['To'] = empfangsEmail
    msg['Subject'] = "Finaler Auftrag vom Nordpol"

    emailText = "Hallo "+Wichtel  +" nach einigen technischen Schwierigkeiten ist dies nun dein endgültiger Auftrag für Weihnachten. Denn wie du sicherlich weißt, wurde auf dem Nordpol 2g Regelungen am Arbeitsplatz eingeführt. Da viele meiner Mitarbeiter noch nicht geimpft sind können diese nun nicht Arbeiten kommen. Homeoffice ist auch schwierig in meiner Branche. Daher würde ich dich bitten, für" + empfaenger+" eine Kleinigkeit für das Weihnachtsfest im Hause Beiker/Ernst mitzubringen. Vielen dank im vorraus. Dein Weihnachtsmann ps. Wenn es jetzt auch nicht funktioniert hat wird das ganze halt Analog gemacht ich bin sowieso nicht überzeugt von diesem ganzen Technick schnick schnack"
    msg.attach(MIMEText(emailText, 'html'))

    server = smtplib.SMTP('smtp.gmail.com', 587)  # Die Server Daten
    server.starttls()
    server.login(senderEmail, "Willi5366")  # Das Passwort
    text = msg.as_string()
    server.sendmail(senderEmail, empfangsEmail, text)
    server.quit()
    print("Email sent")

Wichtelliste=[("Henry","HenryBeiker@aol.com"),("Ellen","ellen.beiker@aol.com"),
("Katja","KatjaBeiker@aol.com"),("Thomas","thomsen69@arcor.de"),
    ("Janina", "janinaernst@yahoo.de"), ("Micha", "michael.massow@outlook.com"), ("Simon", "simon@simon-steigner.de"), ("Alina", "alinaschmitthenner@aol.com")]
empfanger= Wichtelliste

while (x< 8):
    Wichtel = random.sample(Wichtelliste,1)
    empfaenger= random.sample(empfanger,1)
    if (Wichtel[0][0]!=empfaenger[0][0]):
        #wichtelmail(Wichtel[0][0],Wichtel[0][1],empfaenger[0][0]) #auskommentieren zum abschicken der mails
        #print("Wichtel", Wichtel[0][0], "mit Mail", Wichtel[0][1], "schenkt", empfaenger[0][0])
        Wichtelliste= [elem for elem in Wichtelliste if elem not in Wichtel]
        empfanger=[elem for elem in empfanger if elem not in empfaenger]
        x=x+1
        #print(x,Wichtel,empfaenger)
print("Alles abgeschickt")