from . import posts
from .forms import PostForm
from flask import request, redirect, url_for, render_template, flash
from app.models import db, Post
from flask_login import current_user, login_required

# Create a post
@posts.route('/create_post', methods=['GET', 'POST'])
def create_post():
    form = PostForm()
    if request.method == 'POST' and form.validate_on_submit():

        # Grabbing our post form data
        title = form.title.data
        caption = form.caption.data
        img_url = form.img_url.data
        user_id = current_user.id

        # Creating an instance of the Post Model
        new_post = Post(title, caption, img_url, user_id)

        # Adding new post to our database
        db.session.add(new_post)
        db.session.commit()

        flash(f'Successfully created post {title}!', 'success')
        return redirect(url_for('main.home'))
    else:
        return render_template('create_post.html', form=form)

# Viewing all posts
@posts.route('/feed')
def feed():
    posts = Post.query.all()
    return render_template('feed.html', posts=posts)

# Update a post
@posts.route('/update/<int:post_id>', methods=['GET', 'POST'])
@login_required
def update_post(post_id):
    post = Post.query.get(post_id)
    form = PostForm()

    if current_user.id != post.user_id:
        flash("You cannot update someone else's post you snake! 🐍 ", 'warning')
        return redirect(url_for('posts.feed'))

    if request.method == 'POST' and form.validate_on_submit():
        # form data
        title = form.title.data
        caption = form.caption.data
        img_url = form.img_url.data

        # update post object in db
        post.title = title
        post.caption = caption
        post.img_url = img_url

        # commit changes to db
        db.session.commit()

        return redirect(url_for('posts.feed'))
    else:
        return render_template('update_post.html', post=post, form=form)

# delete post
@posts.route('/delete/<int:post_id>')
@login_required
def delete_post(post_id):
    post = Post.query.get(post_id)

    if current_user.id != post.user_id:
        flash("You cannot delete someone else's post you snake! 🐍 ", 'warning')
        return redirect(url_for('posts.feed'))

    db.session.delete(post)
    db.session.commit()
    flash(f'Successfully deleted post {post.title}!', 'info')
    return redirect(url_for('posts.feed'))


# Copyright (C) 2012-2014 Geoscience Australia

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
 Title: vulnerability_model.py

Author:   Ben Cooper, ben.cooper@ga.gov.au

Description: TODO:

Copyright 2012 by Geoscience Australia

"""

import os
import sys
import numpy
import logging
from numpy import asarray, interp, where, zeros

from hazimp.validator import Validator, NRML_SCHEMA
from hazimp.xml_interface import XmlLayer

LOGGER = logging.getLogger(__name__)
DEFAULTLOSS = 0

def vuln_sets_from_xml_file(filenames: list) -> dict:
    """
    Load a GEM NRML vulnerability file in the format described in
    resources/nrml/schema/risk/vulnerability.xsd

    Args:
    :param filenames: The names of the xml files.

    :returns: A dictionary of Vulnerability Sets.
    """

    vulnerability_sets = {}

    for filename in filenames:
        if not os.path.exists(filename):
            raise RuntimeError(
                f'No vulnerability XML was loaded. Check file name {filename}'
            )

        validator = Validator(NRML_SCHEMA)
        try:
            validator.validate(filename)
        except AssertionError as err:
            LOGGER.exception(f"{filename} is not a valid vulnerability file")
            LOGGER.exception(err)
            sys.exit(1)

        xml_node = XmlLayer(filename=filename)
        set_id, vulnerability_set = vuln_sets_from_xml_node(xml_node)
        vulnerability_sets[set_id] = vulnerability_set

    return vulnerability_sets

def vuln_sets_from_xml_node(xml_node):
    """
    Load in the vulnerability sets from an xml node.

    :param vulnerability_models: Dictionary to store vulnerability models.
    :param xml_node: The root node of the vulnerability xml file.

    :returns: A dictionary of Vulnerability Sets.
    """

    xml_vuln_set = xml_node['vulnerabilityModel'][0]
    vuln_set_id = xml_vuln_set.attributes['id']
    asset_category = xml_vuln_set.attributes['assetCategory']
    loss_category = xml_vuln_set.attributes['lossCategory']
    iml = xml_vuln_set['imls'][0]
    im_level = iml.array[0]
    intensity_measure_type = iml.attributes['imt']
    try:
        default_loss = xml_vuln_set.attributes['defaultLoss']
    except KeyError:
        default_loss = DEFAULTLOSS

    vulnerability_functions = {}

    for func in xml_vuln_set['vulnerabilityFunction']:
        vuln_funct = VulnerabilityFunction.from_xml_node(func)
        vulnerability_functions[vuln_funct.function_id] = vuln_funct

    return (vuln_set_id, VulnerabilitySet(
        im_level,
        intensity_measure_type,
        vuln_set_id,
        asset_category,
        loss_category,
        vulnerability_functions,
        default_loss))

class VulnerabilitySet(object):

    """
    A set of vulnerability functions for a given intensity measure
    level.  All vulnerability functions have the same intensity
    measure (x-axis) and loss category (y-axis). Each vulnerability
    function represents a class of assets, such as brick buildings
    etc.

    Methods:
    - calc_mean - return mean loss and sigma for the given function id
    - sample - return a sample for the given function id

    Constructor input:
    - intensity_measure_level - a set of points for the x axis of the mean loss
      curve (common to all functions)
    - intensity_measure_type - type of intensity measure that the intensity
      measure level specifies ('MMI' is the only supported value)
    - vulnerability_functions - a dictionary of VulnerabilityFunction objects
      where the function id is the key

    Class method:
    - from_xml - construct a Vulnerability_Set object from a NRML vulnerability
      file

    """
    # pylint: disable=R0913

    def __init__(self,
                 intensity_measure_level,
                 intensity_measure_type,
                 vulnerability_set_id,
                 asset_category,
                 loss_category,
                 vulnerability_functions,
                 default_loss):
        """
        :params intensity_measure_level: a set of points for the x axis
                of the mean loss curve (common to all functions)
        :params intensity_measure_type: type of intensity measure that the
                intensity measure level specifies e.g. MMI
        :params asset_category: The asset typology
        :params loss_category: The type of loss suffered by the asset_category
        :params vulnerability_functons: a dictionary of VulnerabilityFunction
                 objects where the function id is the key.

        """
        self.intensity_measure_level = asarray(intensity_measure_level)
        self.intensity_measure_type = intensity_measure_type
        self.vulnerability_set_id = vulnerability_set_id
        # asset_category: The asset typology
        # e.g. population, buildings
        self.asset_category = asset_category
        # loss_category: The type of loss suffered by the asset_category
        # e.g. fatalities, collapse, building damage index,
        # contents damage index
        self.loss_category = loss_category
        self.vulnerability_functions = vulnerability_functions
        self.default_loss = default_loss

    def __repr__(self):
        return ('Discrete Vulnerability Set:\n'
                '          intensity measure type: %s\n'
                '         intensity measure level: %s\n'
                'discrete vulnerability functions: %s\n'
                % (str(self.intensity_measure_type),
                   str(self.intensity_measure_level),
                   str(list(self.vulnerability_functions.keys()))))

    def build_realised_vuln_curves(self, vulnerability_function_ids,
                                   variability_method=None):
        """
        Given a list of vulnerability_function_IDs return the
        actual vulnerability curves, as a realised vulnerabitly
        curves instance.

        Currently this just returns a mean curve.

        :parmas vulnerability_function_IDs: A list of the vuln. functions.
            The list dimension is asset.
        :parmas variability_method: How the vulnerability function is sampled.

        :returns: A realised vulnerabitly curves instance.  Use this to calc
            the loss ratio.

        """
        loss_per_asset = []  # Build a list up.
        for key in vulnerability_function_ids:
            try:
                vuln_funct = self.vulnerability_functions[key]
            except KeyError:
                msg = '[%s] does not have a vulnerability curve.\n The' % key
                msg += ' vulnerability set is %s' % self.vulnerability_set_id
                raise NotImplementedError(msg)
            loss_per_asset.append(vuln_funct.get_loss(variability_method))
        # To get dimensions (asset, loss)
        loss_per_asset = asarray(loss_per_asset)
        realised_vuln_curves = RealisedVulnerabilityCurves(
            self.intensity_measure_type,
            self.loss_category,
            self.intensity_measure_level,
            loss_per_asset,
            self.vulnerability_set_id,
            self.default_loss)
        return realised_vuln_curves

    def calc_mean(self, func_id, intensity):
        """
        A wrapper for `VulnerabilityFunction.calc_mean`. Lookup the
        specified vulnerability function in the set and run `calc_mean`
        for that function.

        """
        func = self.vulnerability_functions.get(func_id)
        if func is None:
            raise NotImplementedError(
                '%s does not have a configured vulnerability curve' % func_id)

        return func.calc_mean(intensity, self.intensity_measure_level)

    def sample(self, func_id, mean, sigma):
        """
        A wrapper for VulnerabilityFunction.sample. Lookup the
        specified vulnerability function in the set and run sample for
        that function.

        :param str func_id: Then name of the vulnerability function set
        :param float mean: Mean value of the selected vulnerability function
        :param float sigma: the coefficient of variation of the selected
          vunlerability function

        """
        func = self.vulnerability_functions.get(func_id)
        if func is None:
            raise NotImplementedError(
                '%s does not have a configured vulnerability curve' % func_id)

        return func.sample(mean, sigma)

def ratio_cutoff(ratio):
    """
    As VulnerabilityFunction.sample may return values outside of a ratio,
    ensure that the given ratio array is between 0 and 1 by setting values
    outside these bounds to the bounds.

    :param float ratio: the value to be tested.
    :returns: 1 if ratio > 1.0, 0 if ratio < 0.0, or the value of ratio
     otherwise.

    """
    # High cutoff
    ratio = where(ratio > 1.0, 1.0, ratio)
    # Low cufoff
    ratio = where(ratio < 0.0, 0.0, ratio)

    return ratio

class VulnerabilityFunction(object):

    """
    A vulnerability function defined by a specified set of points on a curve.

    Methods:
    calc_mean - return mean loss and sigma based on the given set of points
    sample - return a sample based on the specified probabilistic distribution

    """

    def __init__(self,
                 func_id,
                 mean_loss,
                 coefficient_of_variation,
                 distribution):
        """
        :param func_id: identifier for this function
        :param mean_loss: array of ratio points
        :param coefficient_of_variation: array of uncertainty points
                (shape must match shape of mean_loss)
        :param distribution: either normal ('N') or lognormal ('LN')
        :param vulnerability_set: The vulnerability set for this function.
        """

        self.function_id = func_id
        self.mean_loss = asarray(mean_loss)
        self.coefficient_of_variation = asarray(coefficient_of_variation)
        self.distribution = asarray(distribution)

    def get_loss(self, variability_method=None):

        """
        Get the actual loss for a curve.

        Currently this just returns a mean curve.

        :param variability_method: How the vulnerability function is sampled.

        :returns: The loss y-axis values, sampled using the supplied method.
        """
        if variability_method is None:
            loss_points = self.mean_loss
        elif variability_method == 'mean':
            loss_points = self.mean_loss
        elif variability_method == 'normal':
            loss_points = self.mean_loss + numpy.random.normal(0, 1) * \
                (self.mean_loss*self.coefficient_of_variation)
        elif variability_method == 'normal_uniform':
            loss_points = self.mean_loss + numpy.random.normal(0, 1) * \
                (self.coefficient_of_variation)
        else:
            raise RuntimeError(("Invalid vulnerability_method in "
                                "configuration file. vulnerability_method "
                                "must be 'normal','normal_uniform, 'mean' "
                                "or left blank"))

        # TODO: Fix!
        return ratio_cutoff(loss_points)

    def calc_mean(self, intensity):
        """
        Calculate mean loss ratio and sigma based on the specified points on
        the curve:
                        |
                        |                                +
                        |                           +
        Mean loss ratio |                     +
                        |               +
                        |            +
                        |          +
                        |         +
                        |        +
                        |       +
                        |    +
                        | +
                        +-----------------------------------
                               Intensity measure level

        For a given intensity, mean loss and sigma is determined by linearly
        interpolating the points on the curve.

        Note that sigma is calculated as cv * mean loss as cv = mean loss/sigma
        """
        mean_loss = interp(intensity,
                           self.intensity_measure_level,
                           self.mean_loss)
        cv = interp(intensity,
                    self.intensity_measure_level,
                    self.coefficient_of_variation)
        # cv = sigma / mean
        sigma = cv * mean_loss

        return (mean_loss, sigma)

    @classmethod
    def from_xml_node(cls, xml_node):
        """Load in a vulnerability function from an xml node.

        :param xml_node: The root node of the vulnerability xml file.

        :returns: A vulnerability function.
        """
        func_id = xml_node.attributes['id']
        prob_dist = xml_node.attributes['dist']
        loss = xml_node['meanLRs'][0].array[0]
        coeff_of_variation = xml_node['covLRs'][0].array[0]

        return cls(
            func_id,
            loss,
            coeff_of_variation,
            prob_dist)

class RealisedVulnerabilityCurves(object):

    """
    Respresents a collection of vulnerability curves associated with an
    exposure set.

    There is one vulnerability curve per asset
    """
    # pylint: disable=R0913

    def __init__(self,
                 intensity_measure_type,
                 loss_category_type,
                 intensity_measure_level,
                 loss_per_asset,
                 vulnerability_set_id,
                 default_loss):
        """

        :param intensity_measure_type: type of intensity measure that the
                intensity measure level specifies ('MMI' is the only supported
                 value).
        :param loss_category_type: Type of loss suffered by the asset_category.
        :param intensity_measure_level: a set of points for the x axis of
            the mean loss curve (common to all functions).
        :param loss_per_asset: 2D array of ratio points per asset.
                The loss dimension can be regarded as the y axis values.
                dimensions (asset, loss).
        """

        self.intensity_measure_type = intensity_measure_type
        self.loss_category_type = loss_category_type
        self.loss_per_asset = loss_per_asset

        self.intensity_measure_level = intensity_measure_level
        self.vulnerability_set_id = vulnerability_set_id
        self.default_loss = default_loss

    def look_up(self, intensity):
        """
        Given an intensity use the curve to determine the loss ratio.

        :param intensity: An array intensity measures.  Dimensions(asset, ...)

        :return: A loss value. loss_category_type describes type of loss.
        """
        # Note the dimensions after the asset are different in intensity
        # and loss_per_asset.

        loss = zeros(intensity.shape)
        assert self.loss_per_asset.shape[0] == intensity.shape[0]

        nan = numpy.isnan(intensity)
        for asset in range(self.loss_per_asset.shape[0]):
            loss[asset] = interp(
                intensity[asset],
                self.intensity_measure_level,
                self.loss_per_asset[asset])
        loss[nan] = self.default_loss
        return loss


# -*- coding: utf-8 -*-

# 猎聘的spider，爬取job和company信息

__author__ = 'bitfeng'

import re
import scrapy
from chinahr.items import JobInfoItem, ComInfoItem
from scrapy.loader import ItemLoader
from chinahr.processors import *
import urllib

# 猎聘网spider
class LiepinCrawlSpider(scrapy.Spider):
    name = 'liepin'  # spider名称
    allowed_domain = ['liepin.com']  # 限制域名
    start_urls = ['http://www.liepin.com/it/?imscid=R000000030',   # 起始urls
                  'http://www.liepin.com/realestate/?imscid=R000000031',
                  'http://www.liepin.com/financial/?imscid=R000000032',
                  'http://www.liepin.com/consumergoods/?imscid=R000000033',
                  'http://www.liepin.com/automobile/?imscid=R000000034',
                  'http://www.liepin.com/medicine/?imscid=R000000054',
                  ]

    # 处理起始urls的response
    def parse(self, response):
        urls = response.xpath('//ul[@class="sidebar float-left"]/li/dl/dd/a/@href').extract()
        for url in urls:
            yield scrapy.Request('http://www.liepin.com'+re.sub(re.compile(u'&dqs=\d*'), '', url), callback=self.parse_flip)

    def parse_flip(self, response):
        category = ''.join(re.compile(u'(?<=&key=).*?(?=&)').findall(response.url))
        urls = response.xpath('//ul[@class="sojob-result-list"]/li/a/@href').extract()
        for url in urls:
            yield scrapy.Request(url, callback=self.parse_info, meta={'category': category})
        reg_url = ''.join(response.xpath('//div[@class="pagerbar"]').
                          re(u'(?<=)\d{1,3}.*?(?=)'))
        next_url = ''.join(re.compile(u'(?<=href=")http.*?(?=")').findall(reg_url))
        if next_url:
            yield scrapy.Request(next_url, callback=self.parse_flip)

    # 抓取 职位信息 和 公司信息
    def parse_info(self, response):
        loaderJob = ItemLoader(item=JobInfoItem(), response=response)
        loaderJob.add_value('url', value=response.url)
        loaderJob.add_value('job_category', value=urllib.unquote(response.meta['category']))
        loaderJob.add_xpath('job_name', '//div[@class="title-info over"]/h1/text()')
        loaderJob.add_xpath('job_name', '//div[@class="title-info "]/h1/text()')
        loaderJob.add_xpath('job_company', '//div[@class="title-info over"]/h3/text()')
        loaderJob.add_xpath('job_company', '//div[@class="title-info "]/h3/text()')
        loaderJob.add_xpath('job_company', '//div[@class="title-info "]/h3/a/text()')
        loaderJob.add_xpath('job_miniEdu', '//div[@class="resume clearfix"]/span/text()', TakeNumL(0))
        loaderJob.add_xpath('job_experience', '//div[@class="resume clearfix"]/span/text()', TakeNumL(1))
        loaderJob.add_xpath('job_reqLan', '//div[@class="resume clearfix"]/span/text()', TakeNumL(2))
        loaderJob.add_xpath('job_reqAge', '//div[@class="resume clearfix"]/span/text()', TakeNumL(3))
        loaderJob.add_xpath('job_salary', '//p[@class="job-main-title"]/text()', TakeFirstL())
        loaderJob.add_xpath('job_location', '//p[@class="basic-infor"]/span[1]/text()', TakeFirstL())
        loaderJob.add_xpath('job_update', '//p[@class="basic-infor"]/span[2]/text()', TakeFirstL(), re=u'(?<=发布于：).*')
        loaderJob.add_xpath('job_desc', '//div[@class="content content-word"][1]', RemoveTagsL(), StripBlankL(), JoinL(''))
        loaderJob.add_xpath('job_benefits', '//div[@class="job-main main-message"]',
                            RemoveTagsL(), ReplaceBlank(), re=u'(?<=薪酬福利：)[\s\S]*')
        loaderJob.add_xpath('job_benefits', '//div[@class="tag-list clearfix"]/span/text()', JoinL('|'))
        yield loaderJob.load_item()

        if 'job.liepin.com' in response.url:
            loaderCom = ItemLoader(item=ComInfoItem(), response=response)
            loaderCom.add_value('url', value=response.url)
            loaderCom.add_value('com_name', value=loaderJob.get_collected_values('job_company'))
            loaderCom.add_xpath('com_industry', '//div[@class="right-post-top"]/div[@class="content content-word"]/a[1]/@title', TakeFirstL())
            loaderCom.add_xpath('com_size', '//div[@class="right-post-top"]/div[@class="content content-word"]', RemoveTagsL(), re=u'(?<=规模：)[\s\S]*?(?=
)')
            loaderCom.add_xpath('com_nature', '//div[@class="right-post-top"]/div[@class="content content-word"]', RemoveTagsL(), re=u'(?<=性质：)[\s\S]*?(?=
)')
            loaderCom.add_xpath('com_address', '//div[@class="right-post-top"]/div[@class="content content-word"]', RemoveTagsL(), re=u'(?<=地址：)[\s\S]*')
            loaderCom.add_xpath('com_intro', '//div[@class="job-main main-message noborder "]/div[@class="content content-word"]/text()', StripBlankL(), TakeFirstL())
            yield loaderCom.load_item()


import json
import random
import numpy as np
import os
from collections import deque

from tensorflow.python.keras.models import Sequential, model_from_json
from tensorflow.python.keras.layers import Dense
from tensorflow.python.keras.optimizers import Adam

# Just disables the warning, doesn't enable AVX/FMA
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

class DQNAgent:
    def __init__(self, model, epsilon):
        """Create an agent using a model. Typically you want to call either from_stored_model or from_dimensions."""
        self.memory = deque(maxlen=250000)
        self.gamma = 0.5  # discount rate
        self.epsilon = epsilon
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.995
        self.learning_rate = 0.001
        self.batch_size = 64
        model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
        self.model = model
        self.input_size = int(self.model.input.shape[-1])
        self.output_size = self.model.output.shape[-1]

    @classmethod
    def from_stored_model(cls, model_file):
        model_and_settings = json.load(open(model_file))
        epsilon = model_and_settings['epsilon']
        model_json = model_and_settings['model']
        model = model_from_json(json.dumps(model_json))

        return cls(model, epsilon)

    @classmethod
    def from_dimensions(cls, state_size, layers, action_size):
        """ Neural Net for Deep-Q learning Model--->>  #Q=NN.predict(state)"""
        model = Sequential()
        # First layer
        model.add(Dense(layers[0], input_dim=state_size, activation='relu'))
        # Middle layers (if any)
        for layersize in layers[1:]:
            model.add(Dense(layersize, activation='relu'))
        # Output layer
        model.add(Dense(action_size, activation='linear'))

        return cls(model, epsilon=0.8)

    def remember(self, state, action, reward, next_state):
        """Store a memory

        Args:
            state: the state at the beginning of the action
            action: the chosen action based on this
            reward: the reward for the action
            next_state: the resulting state or None if the creature died.
        """
        self.memory.append((state, action, reward, next_state))

    def act(self, state):
        """
        Return an action given the state using the internal network.

        Args:
            state: flat array containing the state

        Returns:
            integer indicating the action

        """
        if np.random.rand() <= self.epsilon:
            return random.randrange(self.output_size)
        act_values = self.predict(state)
        return np.argmax(act_values[0])

    def predict(self, state):
        state = np.reshape(state, (1, -1))
        return self.model.predict(state)

    def fit(self, state, target_f):
        state = np.reshape(state, (1, -1))
        self.model.fit(state, target_f, epochs=1, verbose=0)

    def replay(self):
        # Sample a batch from memory uniformly at random
        batch_size = min(self.batch_size, len(self.memory))
        batch = random.sample(self.memory, batch_size)

        # Predict q_values in batches for efficiency
        none_state = np.zeros(self.input_size)  # Used in place of None for next_state
        states = np.array([sample[0] for sample in batch])
        next_states = np.array([(none_state if sample[3] is None else sample[3]) for sample in batch])
        q_values = self.model.predict(states)
        q_values_next = self.model.predict(next_states)

        loss = 0.0

        # Fill in our training batch
        X = np.zeros((batch_size, self.input_size))
        y = np.zeros((batch_size, self.output_size))
        for i in range(batch_size):
            state, action, reward, next_state = batch[i]
            # Important : target is the q_value itself for all actions except the one actually taken
            if next_state is not None:
                reward += self.gamma * np.amax(q_values_next[i])
            target = q_values[i]
            loss += abs(target[action] - reward)
            target[action] = reward
            X[i] = state
            y[i] = target

        self.model.fit(X, y, verbose=0)
        self.epsilon = min(self.epsilon_decay * self.epsilon, self.epsilon_min)

        return loss / batch_size

    def identity_test(self):
        """Run the network over inputs with each exactly one cell set to one."""
        inputs = np.identity(self.input_size)
        return self.model.predict(inputs)

    def show_weights(self):
        for idx, layer in enumerate(self.model.layers):
            print(f'\nLayer {idx} weights (rows are inputs, columns are outputs:')
            print(layer.get_weights()[0])
            print(f'Layer {idx} biases:')
            print(layer.get_weights()[1])
        print()

    def load_weights(self, name):
        self.model.load_weights(name)

    def save_weights(self, name):
        self.model.save_weights(name)

    def save_history(self, name):
        with open(name, 'w') as fout:
            for state, action, reward, next_state in self.memory:
                if next_state is not None:  # Added but not sure why this was not needed in the first place
                    state = [float(x) for x in state]
                    if next is not None:
                        next_state = [float(x) for x in next_state]
                    record = {'state': state, 'action': int(action), 'reward': float(reward), 'next_state': next_state}
                    fout.write(json.dumps(record) + '\n')

    def save_model(self, name):
        model_json = json.loads(self.model.to_json(indent=2))
        with open(name, 'w') as json_file:
            json.dump({'model': model_json, 'epsilon': self.epsilon}, json_file, indent=2)


from urllib.request import urlopen
import os
import sys
import json
import traceback

_resources_dir = 'resources'
_licenses_json_file = 'licenses.json'
_frequentLicenselist_file = 'frequentLicenselist.json'
_frequent_lic_nick_json_file = 'frequent_license_nick_list.json'

def get_license_from_nick():
    licenses = {}
    licenses_file = os.path.join(_resources_dir, _frequent_lic_nick_json_file)

    try:
        base_dir = os.path.join(sys._MEIPASS, vars(sys.modules[__name__])['__package__'])
    except Exception:
        base_dir = os.path.dirname(__file__)

    file_withpath = os.path.join(base_dir, licenses_file)
    try:
        with open(file_withpath, 'r') as f:
            licenses = json.load(f)
    except Exception as ex:
        print(f"Error to get license from json file : {ex}")

    return licenses

def get_spdx_licenses_json():
    success = True
    error_msg = ''
    licenses = ''
    # licenses : https://github.com/spdx/license-list-data/blob/v3.17/json/licenses.json
    licenses_file = os.path.join(_resources_dir, _licenses_json_file)

    try:
        base_dir = os.path.join(sys._MEIPASS, vars(sys.modules[__name__])['__package__'])
    except Exception:
        base_dir = os.path.dirname(__file__)

    file_withpath = os.path.join(base_dir, licenses_file)
    try:
        with open(file_withpath, 'r') as f:
            licenses = json.load(f)
    except Exception as e:
        success = False
        error_msg = 'Failed to open ' + file_withpath + ': ' + str(e)

    return success, error_msg, licenses

def create_frequentlicenses():
    success = True
    error_msg = ''
    licenses = ''
    frequentLicenses = {}
    _frequentLicenses_key = 'frequentLicenses'

    # spdx_txt_url = _spdx_txt_base_url + version + /text/ + licenseId + '.txt'
    _spdx_txt_base_url = 'https://raw.githubusercontent.com/spdx/license-list-data/'

    try:
        success, error_msg, licenses = get_spdx_licenses_json()
        if success:
            version = licenses['licenseListVersion']
            frequentLicenses[_frequentLicenses_key] = []
            for lic in licenses['licenses']:
                tmp_lic = {}
                tmp_lic["fullName"] = lic['name']
                tmp_lic["shortName"] = lic['licenseId']

                deprecated = lic['isDeprecatedLicenseId']
                if deprecated:
                    spdx_txt_url = _spdx_txt_base_url + 'v' + version + '/text/' + 'deprecated_' + lic['licenseId'] + '.txt'
                else:
                    spdx_txt_url = _spdx_txt_base_url + 'v' + version + '/text/' + lic['licenseId'] + '.txt'

                with urlopen(spdx_txt_url) as url:
                    licenseText = url.read().decode('utf-8')
                tmp_lic["defaultText"] = licenseText

                frequentLicenses[_frequentLicenses_key].append(tmp_lic)

    except Exception as e:
        success = False
        error_msg = 'Failed to open and parse licenses.json'
        print(error_msg, ": ", e)
        print(traceback.format_exc())

    return frequentLicenses, success, error_msg

def main():
    _frequencyLicense = os.path.join(_resources_dir, _frequentLicenselist_file)
    frequentLicenses, success, error_msg = create_frequentlicenses()
    if success:
        with open(_frequencyLicense, 'w', encoding='utf-8') as f:
            json.dump(frequentLicenses, f, sort_keys=True, indent=4)
        print("Created " + _frequentLicenselist_file + " in " + _resources_dir)

if __name__ == '__main__':
    main()


from flask import Flask, jsonify, render_template, Blueprint, request, make_response, redirect, url_for
from . import db
from .models import People, User
from datetime import datetime as dt
routes = Blueprint('routes', __name__)

@routes.route("/")
def hello():
    return render_template('users.jinja2')

@routes.route("/person", methods=['GET'])
def person():
    """Create a user via query string parameters."""
    username = request.args.get("user")
    email = request.args.get("email")
    if username and email:
        existing_user = User.query.filter(
            User.username == username or User.email == email
        ).first()
        if existing_user:
            return make_response(f"{username} ({email}) already created!")
        new_user = User(
            username=username,
            email=email,
            created=dt.now(),
            bio="In West Philadelphia born and raised, \
            on the playground is where I spent most of my days",
            admin=False,
        )  # Create an instance of the User class
        db.session.add(new_user)  # Adds new User record to database
        db.session.commit()  # Commits all changes
        redirect(url_for("person"))
    return render_template("users.jinja2", users=User.query.all(), title="Show Users")

@routes.route("/button")
def button():
    return '<button> Click here </button>'
@routes.route("/addperson")
def addperson():
    return render_template("form.html")
@routes.route("/health")
def health():
    return jsonify({'status': 'ok'})
@routes.route("/personadd", methods=['POST'])
def personadd():
    pname = request.form["pname"]
    color = request.form["color"]
    entry = People(pname, color)
    db.session.add(entry)
    db.session.commit()
    return render_template("index.html")


import sqlite3
import os.path

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
db_path = os.path.join(BASE_DIR, "movies.db")
with sqlite3.connect(db_path) as db:

    cur=db.cursor()

id1 = input("1. kategori")
id2 = input("2. kategori")
id3 = input("3. kategori")
id4 = input("4. kategori")

print("******")
film2= cur.execute('SELECT movie_id from dbkategori WHERE category_id in (?,?,?,?) GROUP by movie_id HAVING COUNT(movie_id)  >=  4',
(id1,id2,id3,id4))

film = cur.fetchall()
list = []

for i in film:
    list.append(i)
    print(i)

"""print(list[0][0])"""
print(len(film))

print("-----")


#https://stackoverflow.com/questions/66724687/how-to-solve-a-429-too-many-requests-when-running-discord-py-bot-on-repl-it
#read if get 429 error
from discord.ext import commands
import discord, os, random, time, requests
from keep_alive import keep_alive
import json, asyncio
from replit import db

def _reset():
  db["lastbump"] = 10
  db["reminded"] = False
  exit()

token = os.environ["token"]
intents=discord.Intents.all()

bot = commands.Bot(command_prefix="!",intents=intents)
bot.remove_command('help')
guilds = [871696913987162112]

class userdata:
  def __init__(self):
    pass

  def getdata(self, username):
    try:
      x = requests.get(f"https://scratchdb.lefty.one/v3/user/info/{username}", timeout=10)
    except Exception as e:
      return ["....","Scratchdb took too long to respond"+str(e)]
    statuscode = x.status_code
    if statuscode != 200:
      return ["...",statuscode]
    x = x.json()
    self.username = x["username"]
    self.id = str(x["id"])
    self.pfp = f"https://cdn2.scratch.mit.edu/get_image/user/{self.id}_90x90.png?v="
    self.rank = x["status"]
    self.joined = x["joined"]
    self.WIWO = x["work"]
    self.AM = x["bio"]
    self.country = x["country"]
    self.followers = x["statistics"]["followers"]
    self.following = x["statistics"]["following"]
    return "done"

def _joke(typeofjoke):
  x = requests.get(f"https://official-joke-api.appspot.com/jokes/{typeofjoke}/random")
  if x.status_code != 200:
    return "An error has occurred. please try again later."
  x = x.json()
  if len(x) == 0:
    return "An error has occurred. please try again later."
  x = x[0]
  return [x["setup"],x["punchline"]]

def _fact():
  x = requests.get("https://uselessfacts.jsph.pl/random.json?language=en")
  if x.status_code != 200:
    return "An error has occurred. please try again later."+str(x.status_code)
  else:
    return x.json()["text"]

user = userdata()

@bot.event
async def on_ready():
  print("Bot is ready!")
  channel = bot.get_channel(877770366741803062)
  while True:
    t = (db["lastbump"] + 2*60*60) - int(time.time()//1)
    if t <= 0:
      if db["reminded"] == False:
        db["reminded"] = True
        await channel.send("Time to bump! use `!d bump` command to bump.")

    await asyncio.sleep(1)

@bot.event
async def on_member_join(member):
  role = discord.utils.get(member.guild.roles, name="member")
  print(bool(role == 875675800886726666))
  await member.add_roles(role)
  await member.send("Hey, this feature is in testing. Please go to #general in scratchers hub and confirm you got this DM. Thanks.")

@bot.command() #name
async def tester(ctx):
  await ctx.send("I do work!")

@bot.command()
async def help(ctx, command=None):
  title = ["Flip a coin","Random Number","Say pong","Roll a dice","Calculator","Scratch User Search","Tell a joke","Tell a fact","Tell bots password"]
  value = ["`!flipcoin`","`!rand <num> <num>` with `<num>` being optional","`!ping`","`!rolldice <num>`where <num> is optional. it specifies number of dice.","`!calc <equation>`","`!stats <username>` where <username> is mandatory","`!joke <type>` where `<type>` is optional.\n Type has 3 options:\n General (default), programming, knock-knock","`!fact`", "`!password`"]
  inline= [False, False, False, False,False,False,False,False,False]
  comms = ["flipcoin","rand","ping","rolldice","calc","stats","joke","fact","password"] #everything in comms should be lowercase

  if command == None:
    embed = discord.Embed(title="Commands", description="All commands must start with the prefix `!`",color=discord.Colour.blue())
    for a, b, c in zip(title,value,inline):
      embed.add_field(name = a ,value = b ,inline = c)

  else:
    if not command.lower() in comms:
      embed = discord.Embed(title="Command not found", description=f"{command} was not found. type `!help` to see all commands",color=discord.Colour.red())

      embed.add_field(name="not found...",value=f"{command} not found. Type `!help` to see all commands.")
    else:
      indexcomms = comms.index(command.lower())
      embed = discord.Embed(title=title[indexcomms], description=value[indexcomms],color=discord.Colour.dark_green())

  embed.set_footer(text="`!help <command>` will help you find only the specific command's info. example: `!help flipcoin`")
  await ctx.send(embed=embed)

@bot.command()
async def ping(ctx):
  await ctx.send("Pong")

@bot.command()
async def sleep(ctx):
    await ctx.send("Thanks! bye ðŸ‘‹ zzzzz....")
    await asyncio.sleep(10)
    await ctx.send("Im back! Thanks for the nap time!")

@bot.command()
async def member(ctx):
  member = await ctx.message.guild.query_members(user_ids=[ctx.author.id])
  member = member[0]
  await member.add_roles(discord.utils.get(member.guild.roles, name="member"))
  await ctx.send(f"{ctx.message.author.mention} your got member role.")

@bot.command()
async def rolldice(ctx, num=1):
  numberdice = random.randint(1,6*num)
  print(type(num))
  await ctx.send(f"You rolled a {numberdice}")

@bot.command()
async def flipcoin(ctx):
	heads_tails = ['Heads', 'Tails']
	choice = random.choice(heads_tails)
	await ctx.send(choice)

@bot.command()
async def rand(ctx, num1=0, num2=10):
  if num2 >= num1:
    random_number = random.randint(int(num1), int(num2))
    await ctx.send(str(random_number))
  else:
    await ctx.send("Syntax: !<num1> <num2> and num1 is bigger or equal to num 2")

@bot.command()
async def joke(ctx,t="general"):
  res = _joke(t)
  if "error" in res:
    await ctx.send(f"{ctx.message.author.mention} {res}.\n type `!help joke` for info on command.")
  else:
    await ctx.send(f"{ctx.message.author.mention}\n{res[0]}\n{res[1]}")

@bot.command()
async def fact(ctx):
  x = _fact()
  await ctx.send(f"{ctx.message.author.mention} {x}")
  """
  with open("test.json","r") as file:
    alljoke = json.load(file)
    alljoke.append(x)
  with open("test.json","w") as file:
    json.dump(alljoke)
  """

@bot.command()
async def password(ctx):
  await ctx.send(f"{ctx.message.author.mention}\nhttps://tenor.com/view/rick-astley-rick-roll-dancing-dance-moves-gif-14097983\nWhat did you expect?")

@bot.command()
async def d(ctx, bump=None):
  if bump == "bump":
    t = (db["lastbump"] + 2*60*60) - int(time.time()//1)
    if t <= 0:
      db["lastbump"] = int(time.time()//1)
      db["reminded"] = False
      t = (db["lastbump"] + 2*60*60) - int(time.time()//1)
      await ctx.send(f"{ctx.message.author.mention} Thanks to bump!\n{t} secs left to bump next time.")
    else:
      await ctx.send(f"Please wait {t} secs before bumping. :)")

@bot.command()
async def stats(ctx,username=None):
  if username == None:
    await ctx.send("Syntax: `!stats <user>` where <user> is mandatory")
  else:
    code = user.getdata(username)
    if "..." in code:
      if code[1] == 404:
        await ctx.send(f"{ctx.message.author.mention} User doesn't exist")
      else:
        await ctx.send(f"{ctx.message.author.mention} An error occurred. please try again")
    elif "...." in code:
      await ctx.send(f"{ctx.message.author.mention} {code[1]}")
    else:
      embed = discord.Embed(title = user.username, description=f"Stats of {user.username} on Scratch \n https://scratch.mit.edu/users/{user.username}",color=discord.Colour.orange())
      embed.set_thumbnail(url=user.pfp)
      embed.add_field(name = "User ID:",value= f"`{user.id}`", inline = True)
      embed.add_field(name = "Country:",value=f"`{user.country}`", inline = True)
      embed.add_field(name = "Followers:",value=f"`{user.followers}`",inline=True)
      embed.add_field(name = "Rank:",value=f"`{user.rank}`", inline = True)
      embed.add_field(name = "Joined:",value=f"`{user.joined}`", inline = True)
      embed.add_field(name = "Following:",value=f"`{user.following}`",inline=True)
      embed.add_field(name = "About Me:",value= f"`{user.AM}`", inline = False)
      embed.add_field(name = "What I'm Working On:",value=f"`{user.WIWO}`", inline = False)
      embed.set_footer(text="Data from https://scratchdb.lefty.one/ created by @datonelefty")
      await ctx.send(embed=embed)

@bot.command()
async def calc(ctx, eqn=None):
  if eqn == None:
    await ctx.send("What??!!")
  else:
    await ctx.send("WIP. gotta calculate "+eqn)

keys = ["rude","online","spam","good","lol"]
ans = ["don't be a fool","yep","Heyheyhey!! ","thx!",":rofl:",]
mentioning = [True, False,True,False,False]
@bot.event
async def on_message(message):
  if bot.user.mentioned_in(message):
    await message.channel.send('Why u pinged me?? I was sleeping :( .\nUse `!sleep` to make me sleep again')

  elif message.mentions and not message.author.bot:
    mentions = ""
    for member in message.mentions: #member.name also exists
      ping = f"<@!{member.id}>"
      if ping in message.content:
        mentions += member.mention + ", "
    if mentions != "":
      await message.channel.send(f"Dont ping {mentions[:-2]}")

  for key, val, ment in zip(keys, ans, mentioning):
    if message.content.lower() == key:
      if ment:
        await message.channel.send(f"{message.author.mention} "+val)
      else:
        await message.channel.send(val)

  await bot.process_commands(message)#it neccesarry for the on_message stuff not to override the commands

keep_alive()
bot.run(token)


from threading import Thread

import requests

from api import NETWORK_EXCEPTIONS, token, async_utils
from api.albums import _create_album_from_json
from api.tracks import _create_track_from_json
from domain.album import Album
from domain.artist import Artist
from utils import retry

@retry(times=3, exceptions=NETWORK_EXCEPTIONS)
def get_artist(artist_id: str) -> Artist:
    json = requests.get(
        url=f'https://api.spotify.com/v1/artists/{artist_id}',
        headers={'Authorization': f'Bearer {token}'}
    ).json()
    return _create_artist_from_json(json)

@retry(times=3, exceptions=NETWORK_EXCEPTIONS)
def get_artists(artist_ids: list[str]) -> list[Artist]:
    json = requests.get(
        url=f'https://api.spotify.com/v1/artists',
        params={'ids': ','.join(artist_ids)},
        headers={'Authorization': f'Bearer {token}'}
    ).json()
    return [_create_artist_from_json(j) for j in json['artists']]

@retry(times=3, exceptions=NETWORK_EXCEPTIONS)
def get_artists_async(artist_ids: list[str], callback) -> Thread:
    return async_utils.async_request(
        'get',
        url=f'https://api.spotify.com/v1/artists',
        params={'ids': ','.join(artist_ids)},
        headers={'Authorization': f'Bearer {token}'},
        callback=lambda r: callback([_create_artist_from_json(j) for j in r.json()['artists']])
    )

@retry(times=3, exceptions=NETWORK_EXCEPTIONS)
def get_artist_albums(artist_id: str, country_code: str = None) -> list[Album]:
    json = requests.get(
        url=f'https://api.spotify.com/v1/artists/{artist_id}/albums',
        params={'market': country_code, 'limit': 50},
        headers={'Authorization': f'Bearer {token}'}
    ).json()
    return [_create_album_from_json(j) for j in json['items']]

@retry(times=3, exceptions=NETWORK_EXCEPTIONS)
def get_artist_top_tracks(artist_id: str, country_code: str):
    json = requests.get(
        url=f'https://api.spotify.com/v1/artists/{artist_id}/top-tracks',
        params={'market': country_code},
        headers={'Authorization': f'Bearer {token}'}
    ).json()
    return [_create_track_from_json(j) for j in json['tracks']]

@retry(times=3, exceptions=NETWORK_EXCEPTIONS)
def get_related_artists(artist_id: str):
    json = requests.get(
        url=f'https://api.spotify.com/v1/artists/{artist_id}/related-artists',
        headers={'Authorization': f'Bearer {token}'}
    ).json()
    return [_create_artist_from_json(j) for j in json['artists']]

def _create_artist_from_json(json: dict) -> Artist:
    return Artist(json['id'], json['name'], json['followers']['total'], json['genres'], json['popularity'])


# 载入selenium相关模组
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
# 建立Chrome Driver的执行档路径
options=Options()
options.chrome_executable_path="D:\practice\python\selenium-start\chromedriver.exe"
# 建立Driver物件实体，用程式操作浏览器运作
driver=webdriver.Chrome(options=options)
driver.maximize_window() # 视窗最大化
driver.get('https://www.ntu.edu.tw/')
driver.save_screenshot("截图.png") # 做网页截图
driver.close()


import pandas as pd
import numpy as np
from fbprophet import Prophet

df = pd.read_csv('./ssq.csv')

# print(df.loc[:,['date','y1']])
y1 = df.loc[:, ['date', 'y1']]
y1.rename(columns={'date':'ds','y1':'y'},inplace = True)

y2 = df.loc[:, ['date', 'y2']]
y2.rename(columns={'date':'ds','y2':'y'},inplace = True)

y3 = df.loc[:, ['date', 'y3']]
y3.rename(columns={'date':'ds','y3':'y'},inplace = True)

y4 = df.loc[:, ['date', 'y4']]
y4.rename(columns={'date':'ds','y4':'y'},inplace = True)

y5 = df.loc[:, ['date', 'y5']]
y5.rename(columns={'date':'ds','y5':'y'},inplace = True)

y6 = df.loc[:, ['date', 'y6']]
y6.rename(columns={'date':'ds','y6':'y'},inplace = True)

y7 = df.loc[:, ['date', 'y7']]
y7.rename(columns={'date':'ds','y7':'y'},inplace = True)

m = Prophet()
m.fit(y1)
future1 = m.make_future_dataframe(periods=1)
forecast1 = m.predict(future1)
print(forecast1[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y1===========')

m = Prophet()
m.fit(y2)
future2 = m.make_future_dataframe(periods=1)
forecast2 = m.predict(future2)
print(forecast2[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y2===========')

m = Prophet()
m = Prophet()
m.fit(y3)
future3 = m.make_future_dataframe(periods=1)
forecast3 = m.predict(future3)
print(forecast3[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y3===========')

m = Prophet()
m.fit(y4)
future4 = m.make_future_dataframe(periods=1)
forecast4 = m.predict(future4)
print(forecast4[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y4===========')

m = Prophet()
m.fit(y5)
future5 = m.make_future_dataframe(periods=1)
forecast5 = m.predict(future5)
print(forecast5[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y5===========')

m = Prophet()
m.fit(y6)
future6 = m.make_future_dataframe(periods=1)
forecast6 = m.predict(future6)
print(forecast6[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y6===========')

m = Prophet()
m.fit(y7)
future7 = m.make_future_dataframe(periods=1)
forecast7 = m.predict(future2)
print(forecast7[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
print('===========end of y7===========')


import time
import pytest
import softest
from pages.yatra_launch_page import LaunchPage
from utilities.utils import Utils
from ddt import ddt, data, unpack, file_data

@pytest.mark.usefixtures("setUp")
@ddt()
class TestSearchAndFlights(softest.TestCase):

    log = Utils.custom_logger()
    ut = Utils()
    #depart_date = ut.find_Date()

    @pytest.fixture(autouse=True)
    def class_obj_setup(self):
        self.lp = LaunchPage(self.driver)

    # this provides the link of this driver to the test case with the help of request
    # @data(("London", "Mum", depart_date, "1 Stop"), ("Mum", "JFK", depart_date, "2 Stop"))
    # @unpack
    # @file_data("../testdata/testdata.json")
    # @file_data("../testdata/testyml.yaml")
    # @data(*Utils.read_data_from_excel("C:\\Python Selenium\\TestAutomationFramework\\testdata\\tdataexcel.xlsx", "Sheet1"))
    @data(*Utils.read_data_from_csv("C:\\Python Selenium\\TestAutomationFramework\\testdata\\testcsv.csv"))
    @unpack
    def test_search_flights_1_stop(self, going_from, going_to, d_date, stop):
        # Flight search main method
        search_flight_res = self.lp.search_flights_main(going_from, going_to, d_date)

        # handling of page scroll with javascript
        self.lp.pageScroll()

        # click on the Filter type
        search_flight_res.filter_fLights_by_stops(stop)

        # Find out all the search flights
        all_stops = search_flight_res.get_all_search_flights_results()
        self.log.info(len(all_stops))

        # verify the filter showing only 1 stops
        self.ut.assertListItemText(all_stops, stop)
        time.sleep(3)

        # providing going from location
        # lp.enterDepartFromLocation("London")

        # providing going to location
        # lp.enterGoingToLocation("new")

        # Select the departure date
        # lp.enterDepartDate(depart_date)

        # click on flight search button
        # lp.enterClickSearch()

        # sf = SearchFlightResults(self.driver)

        # all_stops = sf.wait_for_presence_of_all_elements(By.XPATH,"//span[contains(text()
        # ,'Non Stop') or contains(text(),'1 Stop') or (contains(text(),'2 Stops'))]")


import numpy as np
import pandas as pd
import pystan
import pickle
import arviz as az

fluos = pd.read_csv('marionette_fluo.csv', index_col='time')
ods = pd.read_csv('marionette_od.csv', index_col='time')
all_gates = list(set([i[:-3] for i in fluos.columns.tolist()]))
#gates = [g for g in all_gates if g not in ['blank', 'positive_control', 'negative_control']]
gates = [g for g in all_gates if g not in ['blank']]

for gate in gates:
    print(gate)
    fluo_sel = fluos.loc[:, fluos.columns.str.startswith(gate)]
    od_sel = ods.loc[:, fluos.columns.str.startswith(gate)]
    fluo = fluo_sel.iloc[:,3]
    od = od_sel.iloc[:,3]
    model = """
        functions {
            real[] growth(real t,
                        real[] y,
                        real[] theta,
                        real[] x_r,
                        int[] x_i
                        ) {
                real dydt[1];
                dydt[1] = theta[1] * y[1] * (1-y[1]/theta[2]);
                return dydt;
            }
        }
        data {
            int<lower=1> T;
            //real y0[1];
            real y[T, 1];
            real t0;
            real ts[T];
        }
        transformed data {
            real x_r[0];
            int x_i[0];
        }
        parameters {
            real<lower=0> theta[2];
            real<lower=0> sigma;
            real<lower=0> y0[1];
        }
        model {
            real y_hat[T, 1];
            theta[1] ~ uniform(0, 1);
            theta[2] ~ uniform(0, 2);
            y0[1] ~ uniform(0, 1);
            sigma ~ normal(0, 0.1);
            y_hat = integrate_ode_rk45(growth, y0, t0, ts, theta, x_r, x_i);
            y[,1] ~ normal(y_hat[,1], sigma);
        }
    """

    data = {
        'T': len(od),
        #'n_wells': 1,
        #'y0': [0.02],
        'y': od.values.reshape(-1, 1),
        't0': -20,
        'ts': od.index
    }
    # Compile the model
    sm = pystan.StanModel(model_code=model)
    # Train the model and generate samples
    fit = sm.sampling(data=data, iter=1000, chains=2, n_jobs=-1, verbose=True)
    print(fit)
    summary_dict = fit.summary()
    df = pd.DataFrame(summary_dict['summary'],
                    columns=summary_dict['summary_colnames'],
                    index=summary_dict['summary_rownames'])
    df.to_csv(gate +  '.csv')
    #with open('Stan-' + gate + '-az.pkl', 'wb') as f:
    #    pickle.dump({'model': sm, 'fit': fit}, f)
    data = az.from_pystan(posterior=fit)
    data.to_netcdf(gate + '.nc')


import os
import sys
import numpy as np
import pandas as pd
import torch
from torchvision import transforms
from torch.utils import data
from torch import Tensor

from datetime import datetime
import itertools
import sqlite3
import logging
import time

from dataset.db_query import LeagueTag, SeasonTag
from dataset.dataset import SingleSeasonSingleLeague, SingleSeasonSingleLeagueTimeSlices

"""
All together we have 75 seasons in different leagues (minus the ones
we don't use because they have to many defect entries)

Each Seasons has around 316 matches (~ Bundesliga) probably all together
around 24000 matches ...

We do a 80/20 Train-Test Split where we use some of the Train seasons
for Validation.

i.e.
    50 for Train
    10 for Valid
    15 for Test
"""

TRAIN_SEASONS = [(LeagueTag.NET, SeasonTag.S14_15),
                 (LeagueTag.SPA, SeasonTag.S08_09),
                 (LeagueTag.NET, SeasonTag.S13_14),
                 (LeagueTag.SCO, SeasonTag.S12_13),
                 (LeagueTag.SCO, SeasonTag.S15_16),
                 (LeagueTag.NET, SeasonTag.S10_11),
                 (LeagueTag.BEL, SeasonTag.S15_16),
                 (LeagueTag.ITA, SeasonTag.S08_09),
                 (LeagueTag.ENG, SeasonTag.S09_10),
                 (LeagueTag.BEL, SeasonTag.S09_10),
                 (LeagueTag.NET, SeasonTag.S09_10),
                 (LeagueTag.ITA, SeasonTag.S12_13),
                 (LeagueTag.FRA, SeasonTag.S14_15),
                 (LeagueTag.BEL, SeasonTag.S14_15),
                 (LeagueTag.ITA, SeasonTag.S14_15),
                 (LeagueTag.ITA, SeasonTag.S15_16),
                 (LeagueTag.GER, SeasonTag.S13_14),
                 (LeagueTag.ITA, SeasonTag.S09_10),
                 (LeagueTag.SWI, SeasonTag.S11_12),
                 (LeagueTag.FRA, SeasonTag.S12_13),
                 (LeagueTag.ENG, SeasonTag.S13_14),
                 (LeagueTag.FRA, SeasonTag.S15_16),
                 (LeagueTag.SPA, SeasonTag.S10_11),
                 (LeagueTag.SWI, SeasonTag.S15_16),
                 (LeagueTag.SPA, SeasonTag.S13_14),
                 (LeagueTag.SCO, SeasonTag.S08_09),
                 (LeagueTag.SPA, SeasonTag.S09_10),
                 (LeagueTag.NET, SeasonTag.S11_12),
                 (LeagueTag.SPA, SeasonTag.S14_15),
                 (LeagueTag.ENG, SeasonTag.S14_15),
                 (LeagueTag.SCO, SeasonTag.S10_11),
                 (LeagueTag.ITA, SeasonTag.S11_12),
                 (LeagueTag.SCO, SeasonTag.S14_15),
                 (LeagueTag.POR, SeasonTag.S10_11),
                 (LeagueTag.NET, SeasonTag.S15_16),
                 (LeagueTag.SPA, SeasonTag.S11_12),
                 (LeagueTag.FRA, SeasonTag.S13_14),
                 (LeagueTag.GER, SeasonTag.S14_15),
                 (LeagueTag.GER, SeasonTag.S12_13),
                 (LeagueTag.BEL, SeasonTag.S13_14),
                 (LeagueTag.SCO, SeasonTag.S11_12),
                 (LeagueTag.POR, SeasonTag.S15_16),
                 (LeagueTag.POR, SeasonTag.S12_13),
                 (LeagueTag.GER, SeasonTag.S09_10),
                 (LeagueTag.BEL, SeasonTag.S10_11),
                 (LeagueTag.SWI, SeasonTag.S09_10),
                 (LeagueTag.ITA, SeasonTag.S10_11),
                 (LeagueTag.ENG, SeasonTag.S15_16),
                 (LeagueTag.SWI, SeasonTag.S12_13),
                 (LeagueTag.BEL, SeasonTag.S11_12),
                 (LeagueTag.POR, SeasonTag.S09_10)]

VALID_SEASONS = [(LeagueTag.SCO, SeasonTag.S13_14),
                 (LeagueTag.SWI, SeasonTag.S10_11),
                 (LeagueTag.POR, SeasonTag.S11_12),
                 (LeagueTag.ITA, SeasonTag.S13_14),
                 (LeagueTag.SPA, SeasonTag.S12_13),
                 (LeagueTag.NET, SeasonTag.S12_13),
                 (LeagueTag.POR, SeasonTag.S14_15),
                 (LeagueTag.ENG, SeasonTag.S11_12),
                 (LeagueTag.FRA, SeasonTag.S11_12),
                 (LeagueTag.ENG, SeasonTag.S10_11)]

TEST_SEASONS = [(LeagueTag.FRA, SeasonTag.S09_10),
                (LeagueTag.GER, SeasonTag.S08_09),
                (LeagueTag.BEL, SeasonTag.S12_13),
                (LeagueTag.ENG, SeasonTag.S08_09),
                (LeagueTag.GER, SeasonTag.S11_12),
                (LeagueTag.ENG, SeasonTag.S12_13),
                (LeagueTag.SWI, SeasonTag.S14_15),
                (LeagueTag.FRA, SeasonTag.S08_09),
                (LeagueTag.GER, SeasonTag.S10_11),
                (LeagueTag.SCO, SeasonTag.S09_10),
                (LeagueTag.SPA, SeasonTag.S15_16),
                (LeagueTag.SWI, SeasonTag.S13_14),
                (LeagueTag.GER, SeasonTag.S15_16),
                (LeagueTag.FRA, SeasonTag.S10_11),
                (LeagueTag.POR, SeasonTag.S13_14)]

SMALL_TRAIN_SEASONS = [(LeagueTag.NET, SeasonTag.S14_15),
                       (LeagueTag.SPA, SeasonTag.S08_09),
                       (LeagueTag.NET, SeasonTag.S13_14),
                       (LeagueTag.SCO, SeasonTag.S12_13),
                       (LeagueTag.SCO, SeasonTag.S15_16),
                       (LeagueTag.NET, SeasonTag.S10_11),
                       (LeagueTag.BEL, SeasonTag.S15_16),
                       (LeagueTag.ITA, SeasonTag.S08_09),
                       (LeagueTag.ENG, SeasonTag.S09_10),
                       (LeagueTag.BEL, SeasonTag.S09_10)]

SMALL_VALID_SEASONS = [(LeagueTag.SCO, SeasonTag.S13_14),
                 (LeagueTag.SWI, SeasonTag.S10_11),
                 (LeagueTag.POR, SeasonTag.S11_12)]

SMALL_TEST_SEASONS = [(LeagueTag.FRA, SeasonTag.S09_10),
                (LeagueTag.GER, SeasonTag.S08_09),
                (LeagueTag.BEL, SeasonTag.S12_13)]

def make_dataset(sql_path,
                 season_list,
                 undersample_probs=(1.0, 1.0, 1.0),
                 use_ts=False,
                 ts_size=30,
                 with_odds=False):
    datasets = list()
    for league_tag, season_tag in season_list:
        if use_ts:
            d = SingleSeasonSingleLeagueTimeSlices(
                sql_path,
                league_tag,
                season_tag,
                slice_size=ts_size,
                undersample_probs=undersample_probs,
                load_odds=with_odds)
        else:
            d = SingleSeasonSingleLeague(
                sql_path,
                league_tag,
                season_tag,
                undersample_probs=undersample_probs, load_odds=with_odds)
        datasets.append(d)
    return data.ConcatDataset(datasets)

def make_train_set(sql_path, undersample_probs=(1.0, 1.0, 1.0), use_ts=False, ts_size=30, with_odds=False):
    return make_dataset(
        sql_path,
        TRAIN_SEASONS,
        undersample_probs=undersample_probs,
        use_ts=use_ts,
        ts_size=ts_size,
        with_odds=with_odds)

def make_valid_set(sql_path, undersample_probs=(1.0, 1.0, 1.0), use_ts=False, ts_size=30, with_odds=False):
    return make_dataset(
        sql_path,
        VALID_SEASONS,
        undersample_probs=undersample_probs,
        use_ts=use_ts,
        ts_size=ts_size,
        with_odds=with_odds)

def make_test_set(sql_path, undersample_probs=(1.0, 1.0, 1.0), use_ts=False, ts_size=30, with_odds=False):
    return make_dataset(
        sql_path,
        TEST_SEASONS,
        undersample_probs=undersample_probs,
        use_ts=use_ts,
        ts_size=ts_size,
        with_odds=with_odds)

def make_small_train_set(sql_path,
                         undersample_probs=(1.0, 1.0, 1.0),
                         use_ts=False,
                         ts_size=30, with_odds=False):
    return make_dataset(
        sql_path,
        SMALL_TRAIN_SEASONS,
        undersample_probs=undersample_probs,
        use_ts=use_ts,
        ts_size=ts_size,
        with_odds=with_odds)

def make_small_valid_set(sql_path,
                         undersample_probs=(1.0, 1.0, 1.0),
                         use_ts=False,
                         ts_size=30, with_odds=False):
    return make_dataset(
        sql_path,
        SMALL_VALID_SEASONS,
        undersample_probs=undersample_probs,
        use_ts=use_ts,
        ts_size=ts_size,
        with_odds=with_odds)

def make_small_test_set(sql_path,
                        undersample_probs=(1.0, 1.0, 1.0),
                        use_ts=False,
                        ts_size=30, with_odds=False):
    return make_dataset(
        sql_path,
        SMALL_TEST_SEASONS,
        undersample_probs=undersample_probs,
        use_ts=use_ts,
        ts_size=ts_size, with_odds=with_odds)


from itertools import chain

def simulateDay(fish):
    assert 0 <= fish and fish <= 8

    if fish == 0:
        return [8, 6]
    else:
        return [fish - 1]

def emptyPool():
    fishPool = {}
    for i in range(8 + 1):
        fishPool[i] = 0
    return fishPool

def simulatePool(fishPool):
    newFishpool = emptyPool()
    for i in range(1, 8 + 1):
        newFishpool[i - 1] = fishPool[i]
    newFishpool[6] += fishPool[0]
    newFishpool[8] += fishPool[0]
    return newFishpool

def main():
    with open("input.txt", "r") as f:
        inputFish = [int(x.strip()) for x in f.readline().split(",")]

    fish = inputFish

    print("start:", fish)

    for i in range(80):
        newFish = list(chain.from_iterable(simulateDay(aFish) for aFish in fish))
        fish = newFish

    print("num fish:", len(fish))

    fishPool = emptyPool()
    for aFish in inputFish:
        fishPool[aFish] += 1

    print("fishPool", fishPool)
    for i in range(80):
        fishPool = simulatePool(fishPool)
        print(fishPool)

    print("num fish:", sum(fishPool.values()))

    fishPool = emptyPool()
    for aFish in inputFish:
        fishPool[aFish] += 1

    print("fishPool", fishPool)
    for i in range(256):
        fishPool = simulatePool(fishPool)
        print(fishPool)

    print("num fish:", sum(fishPool.values()))

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Apr  5 10:43:15 2023

@author: Francisco
"""
import cv2
import sys
import numpy as np
import io
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
import threading
from constants import *
import time as t

def get_img_from_fig(fig, dpi=180):
    buf = io.BytesIO()
    fig.savefig(buf, format="png", dpi=dpi)
    buf.seek(0)
    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
    buf.close()
    img = cv2.imdecode(img_arr, 1)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    return img

class Finger():
    def __init__(self,gripper):
        #rospy.init_node('finger_sensor_node', anonymous=True)
        self.avg_time=0.0
        self.frames=0
        self.run_threads=True
        self.sensibility_alpha=SENSIBILITY_ALPHA
        self.sensibility_beta=SENSIBILITY_BETA
        self.center_initial=[]
        self.dot_tracking_img=np.zeros((WIDTH_SCREEN, HEIGHT_SCREEN, 3), dtype = np.uint8)
        self.rgb_image=np.zeros((WIDTH_SCREEN, HEIGHT_SCREEN, 3), dtype = np.uint8)
        self.plot_image=np.zeros((WIDHT_PLOT, HEIGHT_PLOT, 3), dtype = np.uint8)
        self.plot_image_large=None
        self.gripper=gripper
        self.gripper_open=False
        self.ready_to_handover=False
        self.magnitudes=[]
        self.gripper_states=[]
        self.gripper_state=0
        self.plotting=True
        self.ready_to_grasp=False
        self.start=False
        self.fig = Figure(figsize=(5, 4), dpi=100)
        self.camera = threading.Thread(target=self.init_camera_stream)
        self.camera.start()
        self.plotter = threading.Thread(target=self.plot)
        self.plotter.start()
        #self.image_finger_pub_1 = rospy.Publisher("/tactile_sensor_image_1", Image, queue_size=10)
        #self.image_finger_pub_2 = rospy.Publisher("/tactile_sensor_image_2", Image, queue_size=10)
        #self.br=CvBridge()
    def init_camera_stream(self):
        x, y, w, h = X_OFFSET, Y_OFFSET, WIDTH_SCREEN, HEIGHT_SCREEN
        cap = cv2.VideoCapture(SENSOR_CAM_ID)
        time=0
        self.magnitudes=[]
        magnitude=0.0
        change_counter=0
        if not cap.isOpened():
            print("Cannot open camera")
            sys.exit(0)
        try:
            #n=0
            while self.run_threads:
                #n+=1
                start = t.time()
                ret, color_image = cap.read()
                color_image_rgb=np.asanyarray(color_image)[x:x+w, y:y+h]
                self.rgb_image=color_image_rgb
                color_image= cv2.bitwise_not(color_image_rgb)
                gray = cv2.cvtColor(color_image, cv2.COLOR_BGR2GRAY)
                (thresh,gray) = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
                hsv = cv2.cvtColor(color_image, cv2.COLOR_BGR2HSV)
                lower = np.array([0,0,122])
                lower = np.array([0,0,155])
                upper = np.array([185,255,255])
                mask = cv2.inRange(hsv, lower, upper)
                hsv = cv2.cvtColor(color_image, cv2.COLOR_BGR2HSV)
                hMin = 0
                sMin = 0
                vMin = 190
                hMax = 179
                sMax =255
                vMax = 255
                lower = np.array([hMin, sMin, vMin])
                upper = np.array([hMax, sMax, vMax])
                mask = cv2.inRange(hsv, lower, upper)
                self.dot_tracking_img = cv2.bitwise_and(color_image,color_image, mask= mask)
                cv2.circle(self.dot_tracking_img, (int(w/2), int(h/2)), 240, (0,0,0), 200)
                magnitude_new=0.0
                if self.start:
                    if time<POINT_TIME:
                        self.ready_to_handover=False
                        self.ready_to_grasp=False
                        output,self.center_initial=tagger_init(self.dot_tracking_img)
                        time+=1
                    else:
                        self.ready_to_handover=True
                        self.ready_to_grasp=True
                        output,magnitude_new=tagger(self.dot_tracking_img,self.center_initial)
                else:
                    time=0
                    self.ready_to_grasp=False
                    magnitude=0.0
                    magnitude_new=0.0
                dev=abs(magnitude_new-magnitude)
                if dev<2.0:
                    dev=0.0
                    change_counter+=1
                    if change_counter==50:
                        if self.gripper_open:
                            self.gripper_state=2
                        else:
                            self.gripper_state=0
                        change_counter=0
                else:
                    self.gripper_state=1
                magnitude=magnitude_new
                if len(self.magnitudes)>QUEUE_LENGHT:
                    self.magnitudes.pop(0)
                    self.gripper_states.pop(0)
                self.magnitudes.append(dev)
                if len(self.magnitudes)>self.sensibility_beta:
                    #print(sum(self.magnitudes[-self.sensibility_beta:])/len(self.magnitudes))
                    if (sum(self.magnitudes[-self.sensibility_beta:])/len(self.magnitudes))>self.sensibility_alpha:
                        #self.gripper_state=2
                        if not self.gripper_open:
                            print("Open gripper")
                            self.gripper.move(10)
                        self.gripper_open=True
                        self.ready_to_grasp=False
                        self.ready_to_handover=False

                self.gripper_states.append(self.gripper_state)
                end = t.time()
                self.avg_time=round((end-start)*1000.0,3)
        finally:
            self.plotting=False
        return
    def plot(self):
        while self.plotting:
            fig = plt.figure()
            fig.set_figwidth(17)
            ax = fig.add_subplot(111)
            ax.plot(self.magnitudes)
            ax.set_title("Tactile sensor dot traking")
            ax.set_ylim(bottom=-0.1)
            img=get_img_from_fig(fig)
            scale_percent = 30
            width = int(img.shape[1] * scale_percent / 100)
            height = int(img.shape[0] * scale_percent / 100)
            dim = (width, height)
            self.plot_image_large=img.copy()
            image= cv2.resize(img, dim, interpolation = cv2.INTER_AREA)
            for index in range(0,len(self.gripper_states)):
                x_i=int(145+index*1.29 - 2)
                y_i=int(35-2)
                x_f=int(145+index*1.29 + 2)
                y_f=int(35 +2)
                if self.gripper_states[index]==0:
                    cv2.rectangle(image, (x_i,y_i), (x_f,y_f), RED, -1)
                elif self.gripper_states[index]==1:
                    cv2.rectangle(image, (x_i,y_i), (x_f,y_f), YELLOW, -1)
                elif self.gripper_states[index]==2:
                    cv2.rectangle(image, (x_i,y_i), (x_f,y_f), GREEN, -1)
            image = cv2.putText(image, str(self.avg_time)+" ms", (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,0), 2, cv2.LINE_AA)
            self.plot_image=image
            del fig
            del img
            del ax
        return


#%%
from PIL import Image
import os

# Files can be downloaded with google-chrome's image downloader plugin

# %%
imagePath = os.path.join(os.getcwd(), 'downloads')
targetPath = os.path.join(os.getcwd(), 'preprocessed')

for i, fileName in enumerate(os.listdir(imagePath)):

    fullImagePath = os.path.join(imagePath, fileName)
    fullTargetPath = os.path.join(targetPath, f"image_{i}.jpg")

    print(f"Editing image {fullImagePath}")

    image = Image.open(fullImagePath)
    imageRgb = image.convert('RGB')
    imageResized = imageRgb.resize((64, 64))
    with open(fullTargetPath, "wb") as f:
        imageResized.save(f)

# %%


import bpy
from bpy.props import *
from .. tree_info import getNodeByIdentifier

class MoveViewToNode(bpy.types.Operator):
    bl_idname = "an.move_view_to_node"
    bl_label = "Move View to Node"
    bl_description = ""

    nodeIdentifier: StringProperty(default = "")
    treeName: StringProperty()
    nodeName: StringProperty()

    @classmethod
    def poll(cls, context):
        try: return context.space_data.node_tree.bl_idname == "an_AnimationNodeTree"
        except: return False

    def execute(self, context):
        searchNode = self.getNode()
        if searchNode is None: return {"CANCELLED"}

        tree = searchNode.id_data

        context.space_data.node_tree = tree
        for node in tree.nodes:
            node.select = False

        searchNode.select = True
        tree.nodes.active = searchNode

        bpy.ops.node.view_selected()
        return {"FINISHED"}

    def getNode(self):
        if self.nodeIdentifier != "":
            try: return getNodeByIdentifier(self.nodeIdentifier)
            except: return None
        else:
            try: return bpy.data.node_groups[self.treeName].nodes[self.nodeName]
            except: return None


from itertools import permutations
from copy import deepcopy

def combinationSum(candidates, target):
    result = []
    combinationSumRecu(sorted(candidates), result, 0, [], target)
    return result

def combinationSumRecu(candidates, result, start, intermediate, target):
    if target == 0:
        result.append(list(intermediate))
    while start < len(candidates) and candidates[start] <= target:
        intermediate.append(candidates[start])
        combinationSumRecu(candidates, result, start, intermediate, target - candidates[start])
        intermediate.pop()
        start += 1

def findSchedules(work_hour, day_hours, pattern):
    pattern = list(pattern)
    total_available = 0
    question_count = 0

    for i in pattern:
        if i != '?':
            total_available += int(i)
        else:
            question_count += 1

    total_available = work_hour - total_available

    if total_available <= 0:
        return [''.join(pattern).replace('?', '0')]

    new_list = []
    for i in range(1, day_hours+1):
        new_list.append(i)

    return_list = combinationSum(new_list, total_available)

    i = 0
    while i < len(return_list):
        if len(return_list[i]) > question_count:
            del return_list[i]
            continue
        elif len(return_list[i]) < question_count:
            return_list[i] += [0]*(question_count - len(return_list[i]))

        i += 1

    final_set = set()

    for i in return_list:
        for j in permutations(i, len(i)):
            final_set.add(j)

    return_list = []
    old_pattern = deepcopy(pattern)

    for schedule in sorted(final_set):
        counter = 0
        pattern = deepcopy(old_pattern)
        for i in range(len(pattern)):
            if pattern[i] == '?':
                pattern[i] = str(schedule[counter])
                counter += 1

        return_list.append(''.join(pattern))
    return return_list

print(findSchedules(20, 8, '??888??'))


import numpy as np
import cv2
import config

all_frames = []
for i, filename in enumerate(config.input_filenames):
    frame = cv2.imread(filename)
    frame = cv2.resize(frame, (config.target_width, config.target_height), interpolation=cv2.INTER_AREA)
    all_frames.append(frame)
    cv2.imwrite(config.frame_path + str(i) + '.png', frame)
all_frames = np.stack(all_frames, axis=0)
np.save(config.frame_path + 'all_frames', all_frames)
print('All Frames Saved: ' + config.frame_path + 'all_frames.npy')

from raft.api import do_match
from fastwarp.api import flow_to_normal as flow_to_normal

def compute_flow(f, t):
    frame = cv2.imread(config.frame_path + str(f) + '.png')
    target_frame = cv2.imread(config.frame_path + str(t) + '.png')
    flow = do_match(frame, target_frame, scale=1.0, iter=20)
    flag = str(f) + 'T' + str(t)
    np.save(config.flow_path + flag, flow)
    cv2.imwrite(config.flow_path + flag + '.png', flow_to_normal(flow, uint8=True))
    print('Flow Computed: ' + flag)
    return flow

all_flows = []
for i in range(config.file_count):
    flows = []
    for t in config.key_flow_range(i):
        flow = compute_flow(i, t)
        flows.append(flow)
    flows = np.stack(flows, axis=0)
    all_flows.append(flows)
all_flows = np.stack(all_flows, axis=0)

np.save(config.flow_path + 'all_flows', all_flows)
print('All Flows Saved: ' + config.flow_path + 'all_flows.npy')


"""
This command is used to list items in a library.

Examples
^^^^^^^^

- List all document files associated will all documents in the library:

    .. code:: sh

        papis list --all --file

    .. raw:: HTML

        <script type="text/javascript"
        src="https://asciinema.org/a/XwD0ZaUORoOonwDw4rXoQDkjZ.js"
        id="asciicast-XwD0ZaUORoOonwDw4rXoQDkjZ" async></script>

- List the year and title of all documents with some custom formatting:

    .. code:: sh

        papis list --all --format '{doc[year]} {doc[title]}'

    .. raw:: HTML

        <script type="text/javascript"
        src="https://asciinema.org/a/NZ8Ii1wWYPo477CIL4vZhUqOy.js"
        id="asciicast-NZ8Ii1wWYPo477CIL4vZhUqOy" async></script>

- List all documents according to the ``bibitem`` formatting (stored in a template
  file ``bibitem.template``):

    .. code:: sh

        papis list --all --template bibitem.template

    .. raw:: HTML

        <script type="text/javascript"
        src="https://asciinema.org/a/QZTBZ3tFfyk9WQuJ9WWB2UpSw.js"
        id="asciicast-QZTBZ3tFfyk9WQuJ9WWB2UpSw" async></script>

- For scripting, printing the id of a series of documents is valuable in order
  to further use the id in other scripts.

    .. code:: sh

        papis_id=$(papis list --id einstein)
        papis open papis_id:${papis_id}
        papis edit papis_id:${papis_id}
        # etc.

Command-line Interface
^^^^^^^^^^^^^^^^^^^^^^

.. click:: papis.commands.list:cli
    :prog: papis list
"""

import os
from typing import List, Optional, Union, Sequence, Tuple

import click

import papis
import papis.id
import papis.utils
import papis.strings
import papis.config
import papis.database
import papis.document
import papis.downloaders
import papis.cli
import papis.pick
import papis.format
import papis.logging

logger = papis.logging.get_logger(__name__)

def run(documents: List[papis.document.Document],
        libraries: bool = False,
        downloaders: bool = False,
        pick: bool = False,
        files: bool = False,
        folders: bool = False,
        papis_id: bool = False,
        info_files: bool = False,
        notes: bool = False,
        fmt: str = "",
        template: Optional[str] = None
        ) -> Sequence[Union[str, papis.document.Document]]:
    """Main method to the list command

    :returns: List different objects
    """
    if downloaders:
        return [str(d) for d in papis.downloaders.get_available_downloaders()]

    if template is not None:
        if not os.path.exists(template):
            logger.error("Template file '%s' not found.", template)
            return []
        with open(template) as fd:
            fmt = fd.read()

    if libraries:
        config = papis.config.get_configuration()
        return [
            section + " " + config[section]["dir"]
            for section in config
            if "dir" in config[section]]

    if files:
        return [
            doc_file for files in [
                document.get_files() for document in documents
            ] for doc_file in files
        ]
    elif papis_id:
        return [papis.id.get(d) for d in documents]
    elif notes:
        return [
            os.path.join(d.get_main_folder() or "", d["notes"])
            for d in documents
            if d.get_main_folder() is not None
            and "notes" in d and isinstance(d["notes"], str)
            and os.path.exists(
                os.path.join(d.get_main_folder() or "", d["notes"]))]
    elif info_files:
        return [d.get_info_file() for d in documents]
    elif fmt:
        return [
            papis.format.format(fmt, document,
                                default=papis.document.describe(document))
            for document in documents
        ]
    elif folders:
        return [
            str(d.get_main_folder()) for d in documents
            if d.get_main_folder() is not None
        ]
    else:
        return documents

@click.command("list")
@click.help_option("--help", "-h")
@papis.cli.query_argument()
@papis.cli.sort_option()
@papis.cli.bool_flag(
    "-i", "--info",
    help="Show the info file name associated with the document")
@papis.cli.bool_flag(
    "--id", "_papis_id",
    help="Show the papis_id")
@papis.cli.bool_flag(
    "-f", "--file", "_file",
    help="Show the file name associated with the document")
@papis.cli.bool_flag(
    "-d", "--dir", "_dir",
    help="Show the folder name associated with the document")
@papis.cli.bool_flag(
    "-n", "--notes",
    help="List notes files, if any")
@click.option(
    "--format", "_format",
    help="List entries using a custom papis format, e.g."
    " '{doc[year] {doc[title]}",
    default="")
@click.option(
    "--template",
    help="Template file containing a papis format to list entries",
    default=None)
@papis.cli.bool_flag(
    "--downloaders",
    help="List available downloaders")
@papis.cli.bool_flag(
    "--libraries",
    help="List defined libraries")
@papis.cli.all_option()
@papis.cli.doc_folder_option()
def cli(query: str,
        info: bool,
        _papis_id: bool,
        _file: bool,
        notes: bool,
        _dir: bool,
        _format: str,
        doc_folder: Tuple[str, ...],
        template: Optional[str], _all: bool, downloaders: bool,
        libraries: bool,
        sort_field: Optional[str], sort_reverse: bool) -> None:
    """List documents' properties"""
    documents: List[papis.document.Document] = []

    if (not libraries and not downloaders
            and not _file and not info and not _dir):
        _dir = True

    if not libraries and not downloaders:
        documents = papis.cli.handle_doc_folder_query_all_sort(query,
                                                               doc_folder,
                                                               sort_field,
                                                               sort_reverse,
                                                               _all)
        if not documents:
            logger.warning(papis.strings.no_documents_retrieved_message)
            return

    objects = run(
        documents,
        libraries=libraries,
        downloaders=downloaders,
        notes=notes,
        files=_file,
        folders=_dir,
        papis_id=_papis_id,
        info_files=info,
        fmt=_format,
        template=template)

    for o in objects:
        click.echo(o)


#!/usr/bin/env python3

from html.parser import HTMLParser
from pkg_resources import parse_version
import re, urllib.request, json, os, argparse, ssl

# get url content & html version parser from tomcat repo

def getUrlContent(url):
  ctx = ssl.create_default_context()
  ctx.check_hostname = False
  ctx.verify_mode = ssl.CERT_NONE
  try:
    fp = urllib.request.urlopen(url, context=ctx)
    mybytes = fp.read()
    fp.close()
    return mybytes.decode("utf8")
  except urllib.error.HTTPError:
    return '{"tags": []}'

class CustomHTMLParser(HTMLParser):
  def __init__(self) -> None:
    self.versionlist = []
    super().__init__()

  def __call__(self, html):
    self.feed(html)
    return self.versionlist

  def handle_data(self, data):
    if re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+", data):
      self.versionlist.append(data[1:-1])

# -----------------
# script arguments

argparser = argparse.ArgumentParser(description='Tomcat version check script')
argparser.add_argument('--registryUrl', default='https://registry:5000')
argparser.add_argument('--registryImage', default='tomcat')
argparser.add_argument('--outputEnvFile', default='update.env')
argparser.add_argument('--forceUpdate', action='store_true')
scriptArgs = argparser.parse_args()

tomcatImage = scriptArgs.registryImage
registryUrl = scriptArgs.registryUrl
outputEnvFile = scriptArgs.outputEnvFile
forceUpdate = scriptArgs.forceUpdate

# getting versions available to download
parser = CustomHTMLParser()
availableVersionList = parser(
  getUrlContent("https://apache-mirror.rbc.ru/pub/apache/tomcat/tomcat-9/")
)

# find latest version from apache mirror
latestAvailableVersion = max(availableVersionList, key=lambda x: parse_version(x), default="0")

# getting versions from registry
registryVersionList = json.loads(
  getUrlContent(f"{registryUrl}/v2/{tomcatImage}/tags/list")
)['tags']

# find latest version from registry
latestRegistryVersion = max(registryVersionList, key=lambda x: parse_version(x), default="0")

# default values to .env file
ENV_IMAGE_NEED_UPDATE = 'False'
ENV_IMAGE_UPDATE_VERSION = latestRegistryVersion
ENV_IMAGE_DOWNLOAD_URL = f"https://apache-mirror.rbc.ru/pub/apache/tomcat/tomcat-9/v{ENV_IMAGE_UPDATE_VERSION}/bin/apache-tomcat-{ENV_IMAGE_UPDATE_VERSION}.tar.gz"

# compare latests...
if not forceUpdate and (parse_version(latestAvailableVersion) <= parse_version(latestRegistryVersion)):
  print(f"""No update needed
------------
Image: {tomcatImage}
Current Tomcat version is latest: {latestRegistryVersion}""")

else:
  ENV_IMAGE_NEED_UPDATE = 'True'
  ENV_IMAGE_UPDATE_VERSION = latestAvailableVersion
  ENV_IMAGE_DOWNLOAD_URL = f"https://apache-mirror.rbc.ru/pub/apache/tomcat/tomcat-9/v{ENV_IMAGE_UPDATE_VERSION}/bin/apache-tomcat-{ENV_IMAGE_UPDATE_VERSION}.tar.gz"
  print(f"""Update needed
------------
Image: {tomcatImage}
Current latest registry Tomcat version: {latestRegistryVersion}
Current latest available Tomcat version: {latestAvailableVersion}""")

# write .env file
envFileString = f"IMAGE_NEED_UPDATE={ENV_IMAGE_NEED_UPDATE}\nIMAGE_UPDATE_VERSION={ENV_IMAGE_UPDATE_VERSION}\nIMAGE_DOWNLOAD_URL={ENV_IMAGE_DOWNLOAD_URL}"
f = open(outputEnvFile, 'w')
f.write(envFileString)
f.close()

print(f"""------------
Now you can use "./{outputEnvFile}" file to decide what to do next.
It contents the following lines:
{envFileString}""")