import utrs as u
import variants as v
import protdomains as pds
import coverage as c
import matplotlib.pyplot as plt
import sys
import pandas as pd

def make_protein_domain_color_file(protein_domain_file, transcript_l, database, sortby_col_l, out_path):

    '''Make a protein domain color file.

    Args:
        | protein_domain_file (str): path to file containing the protein domain information.
        | transcript_l (list of strs): list of Ensembl transcript IDs.
        | database (str): protein domain database.
        | sortby_col_l (list of strs): list of names of columns to sort the DataFrame by.
        | out_path (str): path to write the protein domain color file to.
    '''

    print("make_protein_domain_color_file")
    protein_domain_df = pds.get_protein_domain_df(protein_domain_file, transcript_l, database, sortby_col_l)
    protein_domain_color_s = pds.get_protein_domain_color_s(protein_domain_df)
    protein_domain_color_s.to_csv(out_path)
    print("Written protein domain color file to {0}\n".format(out_path))

def make_exon_coord_file(cov_file, transcript, out_path):

    '''Make an exon coordinate file from a coverage file.

    Args:
        | cov_file (str): path to coverage file.
        | transcript (str): Ensembl transcript ID.
        | out_path (str): path to write the exon coordinate file to.
    '''

    print("make_exon_coord_file")
    cov_df = c.get_cov_df(cov_file, transcript, None, None, [])
    exon_coord_df = c.get_exon_coord_df(cov_df)
    exon_coord_df.to_csv(out_path, index=True)
    print("Written exon coordinate file to {0}\n".format(out_path))

def make_png(transcript_l, title_l, track_l, sample_ll, utr_file_l, exon_coord_file_l, cov_file_l,
             variant_file_l, protein_domain_file_l, protein_domain_color_file, setting_dict, png_file):

    '''Make a png which contains coverage/variants/protein domain tracks for 1 or more transcripts, subject to space limitations.

    Args:
        | transcript_l (list of strs): Ensembl transcript ID(s)
        | title_l (list of strs): title to use for each transcript.
        | track_l (list of strs): list of strings of length 3 which encode whether to generate each of the 3 tracks (coverage, variants, protein domains).
        | sample_ll (list of list of strs): list of lists of sample IDs.
        | utr_file_l (list of strs): list of utr file paths.
        | exon_coord_file_l (list of strs): list of exon coordinate file paths.
        | cov_file_l (list of strs): list of coverage file paths.
        | variant_file_l (list of strs): list of variant file paths.
        | protein_domain_file_l (list of strs): list of protein domain file paths.
        | protein_domain_color_file (str): protein domain color file.
        | setting_dict (dictionary): settings for making the png.
        | png_file (str): path to write the png file to.
    '''

    print("make_png")

    #Check the parameters are well-formed.
    if len(transcript_l) == len(title_l) == len(track_l) == len(sample_ll) == len(utr_file_l) == len(exon_coord_file_l) == len(cov_file_l) == len(variant_file_l) == len(protein_domain_file_l):
        True
    else:
        print("ERROR: Parameters of make_png function which are lists must all be the same length.\n")
        return False

    #Check that the maximum number of tracks is not exceeded.
    num_rows, previous_track = 0, None
    track_gap_dict = dict(zip(["c","v","pd", None],
                              [setting_dict["c_track_gap_rows"], setting_dict["v_track_gap_rows"], setting_dict["pd_track_gap_rows"], 0]))
    for track_s in track_l:
        if track_s[0] == "1":
            num_rows += track_gap_dict[previous_track] + setting_dict["c_track_rows"]
            previous_track = "c"
        if track_s[1] == "1":
            num_rows += track_gap_dict[previous_track] + setting_dict["v_anns_top_rows"] + setting_dict["v_track_rows"] + setting_dict["v_anns_bot_rows"] + setting_dict["v_key_rows"]
            previous_track = "v"
        if track_s[2] == "1":
            num_rows += setting_dict["pd_track_rows"] + track_gap_dict[previous_track]
            previous_track = "pd"
    if num_rows > setting_dict["fig_num_rows"]:
        print("ERROR: PNG figure requires {0} rows but there are only {1}\n".format(num_rows,setting_dict["fig_num_rows"]))
        return False

    #Initialise the figure.
    fig = plt.figure()
    plt.rc('text', usetex=True)

    #Make the axes.
    get_tp_from_exon_bp_l = lambda bp, exon_bp_l: exon_bp_l.index(int(bp)) + 1
    num_rows = setting_dict["fig_num_rows"]
    start_row = 0
    title_1_coords = None

    for i in range(len(transcript_l)):

        print("Transcript: {0}".format(transcript_l[i]))

        #Read in the exon positions and the utrs.
        exon_coord_df = pd.read_csv(exon_coord_file_l[i], index_col="exon")
        strand = "+" if (exon_coord_df["end_bp"] > exon_coord_df["start_bp"]).all() else "-"
        if strand == "+":
            print("Transcription direction: forward")
        elif strand == "-":
            print("Transcript direction: reverse")
        exon_bp_l = []
        for j in range(len(exon_coord_df.index)):
            if strand == "+":
                exon_bp_l.extend(range(exon_coord_df["start_bp"].iloc[j],exon_coord_df["end_bp"].iloc[j]+1))
            elif strand == "-":
                exon_bp_l.extend(range(exon_coord_df["start_bp"].iloc[j],exon_coord_df["end_bp"].iloc[j]-1,-1))
        utr_df = u.get_utr_df(utr_file_l[i], strand, transcript_l[i])
        #Add the transcript positions to utr_df.
        utr_df["start_tp"] = utr_df["start_bp"].apply(func=get_tp_from_exon_bp_l, exon_bp_l=exon_bp_l)
        utr_df["end_tp"] = utr_df["end_bp"].apply(func=get_tp_from_exon_bp_l, exon_bp_l=exon_bp_l)

        #Make the title track
        if i == 0:
            plt.figtext(setting_dict["title_1_fig_x"], setting_dict["title_1_fig_y"], title_l[i], fontsize=setting_dict["title_fontsize"])
            title_1_coords = fig.transFigure.transform((setting_dict["title_1_fig_x"], setting_dict["title_1_fig_y"]))
        else:
            title_track = plt.subplot2grid((num_rows,1), (start_row,0))
            title_track.set_axis_off()
            inv = title_track.transData.inverted()
            title_track.text(inv.transform(title_1_coords)[0], setting_dict["title_2_ax_y"], title_l[i], fontsize=setting_dict["title_fontsize"])
            start_row += setting_dict["t_track_rows"]

        #Make the coverage track.'''
        if track_l[i][0] == "1":
            print("Making coverage track.")
            cov_df = c.get_cov_df(cov_file_l[i], transcript_l[i], None, None, sample_ll[i])
            [bound_l, color_l, edge_color_l] = get_exon_bound_color_l(exon_coord_df, utr_df, strand)
            coverage_track = plt.subplot2grid((num_rows,1), (start_row, 0), rowspan=setting_dict["c_track_rows"])
            start_row += setting_dict["c_track_rows"]
            c.make_track(coverage_track, cov_df, bound_l, color_l, edge_color_l, setting_dict)
            start_row += setting_dict["c_track_gap_rows"]

        #Make the variants track.'''
        variant_track = None
        if track_l[i][1] == "1": #Make the variants track.
            print("Making variant track.")
            variant_df = v.get_variant_df(transcript_l[i], variant_file_l[i])
            variant_df.rename(columns={"pos":"bp"}, inplace=True)
            variant_df["tp"] = variant_df["bp"].apply(func=get_tp_from_exon_bp_l, exon_bp_l=exon_bp_l)
            #variant_df.drop_duplicates(subset=["GENE_prot_change","GENE_DNA_change"], inplace=True)
            variant_df.sort_values(by="tp", inplace=True)
            start_row += setting_dict["v_anns_top_rows"]
            variant_track = plt.subplot2grid((num_rows,1), (start_row,0))
            start_row += setting_dict["v_track_rows"]
            start_row += setting_dict["v_anns_bot_rows"]
            variant_key = plt.subplot2grid((num_rows,1), (start_row,0), rowspan=setting_dict["v_key_rows"])
            start_row += setting_dict["v_key_rows"]
            [bound_l, color_l, edge_color_l] = get_exon_bound_color_l(exon_coord_df, utr_df, strand)
            v.make_track(variant_track, bound_l[-1], bound_l, color_l, edge_color_l, variant_df, setting_dict, variant_key)
            start_row += setting_dict["v_track_gap_rows"]

        #Make the protein domain track.'''
        if track_l[i][2] == "1": #Make the protein domains track.
            print("Making protein domain track.")
            protein_domain_df = pds.get_protein_domain_df(protein_domain_file_l[i], [transcript_l[i]], "Pfam", ["Start","End"])
            protein_domain_track = plt.subplot2grid((num_rows,1), (start_row,0))
            start_row += setting_dict["pd_track_rows"]
            protein_domain_color_df = pd.read_csv(protein_domain_color_file, header=None, names=["Domain", "Color"], index_col="Domain")
            protein_domain_color_s = pd.Series(data=protein_domain_color_df["Color"], index=protein_domain_color_df.index)
            del protein_domain_color_df
            protein_domain_color_s = protein_domain_color_s.apply(lambda x: x if "," not in x else tuple([float(f) for f in x.split(",")]))
            pds.make_track(protein_domain_track, protein_domain_df, utr_df, protein_domain_color_s, setting_dict, variant_track)
            start_row += setting_dict["pd_track_gap_rows"]

    fig.set_size_inches(setting_dict["fig_width_inches"], setting_dict["fig_height_inches"])
    plt.savefig(png_file, dpi=setting_dict["fig_dpi"])
    print("Written {0}.\n".format(png_file))

def get_exon_bound_color_l(exon_coord_df, utr_df, strand):

    '''Get the bounds, colors and edge colors required to generate a color bar that displays the utrs and exons for a transcript.

    Args:
        | exon_coord_df (DataFrame): contains the exon base pair and transcript position coordinates.
        | utr_df (DataFrame): contains the utr base pair and transcript position coordinates.
        | strand (str): specifies whether the transcript is on the positive or negative strand.

    Returns:
        exon_bound_color_ll (list of lists of ints and strs): contains bound_l, the list of bounds, color_l, the list of colors and edge_color_l, the list of edge colors.
    '''

    bound_color_df = pd.concat([exon_coord_df,utr_df])
    bound_color_df.drop(["start_bp","end_bp"], axis=1, inplace=True)
    bound_color_df["utr"].fillna(value="exon", inplace=True)
    bound_color_df.sort_values(by=["start_tp","end_tp","utr"], inplace=True)
    bound_color_df.index = range(len(bound_color_df))
    bound_color_df.index.rename("",inplace=True)
    #print bound_color_df

    for i in range(1,len(bound_color_df)):
        if bound_color_df.loc[i,"end_tp"] == bound_color_df.loc[i-1,"end_tp"]:
            continue
        if bound_color_df.loc[i,"start_tp"] < bound_color_df.loc[i-1,"end_tp"]:
            bound_color_df.loc[i,"start_tp"] = bound_color_df.loc[i-1,"end_tp"] + 1
    bound_color_df.drop_duplicates(["start_tp","end_tp"],inplace=True)
    #print bound_color_df

    bound_color_df["color"] = ["white"] * len(bound_color_df.index)
    exon_color_l = ["red","#6E6E6E"]
    exon_color_i = 0
    for i in bound_color_df.index.tolist():
        #print i
        if bound_color_df.loc[i, "utr"] == "5'" or bound_color_df.loc[i,"utr"] == "3'":
            bound_color_df.loc[i,"color"] = "white"
        else:
            bound_color_df.loc[i,"color"] = exon_color_l[exon_color_i]
            exon_color_i = 1 - exon_color_i

    bound_l = bound_color_df["start_tp"].tolist()
    transcript_len = bound_color_df.iloc[len(bound_color_df.index)-1]["end_tp"]
    if bound_l[-1] != transcript_len:
        bound_l.append(transcript_len)
    color_l = bound_color_df["color"].tolist()

    edge_color_l = ["black" if color == "white" else color for color in color_l]

    exon_bound_color_ll = [bound_l, color_l, edge_color_l]

    return exon_bound_color_ll


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

with open('VERSION', 'r') as fh:
    version = fh.read()

setuptools.setup(
    name="html_generators",
    version=version,
    author="Alex Fischer",
    author_email="alex@quadrant.net",
    description="Functional, streaming approach to HTML generation",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/quadrant-newmedia/html_generators",
    packages=['html_generators'],
    package_dir={'': 'src'},
    include_package_data=True,
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.6',
    install_requires=[],
)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 10 15:38:26 2021

@author: toygar
"""
import random
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from seffaflik.elektrik import uretim
# from sklearn.linear_model import LinearRegression
from calendar import monthrange

def number_of_days_in_month(year, month):
    return monthrange(year, month)[1]

df = pd.read_excel('KURULUGUC2016-2021.xlsx',parse_dates=True)

start = pd.to_datetime('2016-01-01')
end = pd.to_datetime('2021-05-01')
rng = pd.date_range(start, end, freq='H')
len(rng)
rng = rng[0:-1]
len(rng)
rng = pd.DataFrame(rng)
rng = rng.set_axis(['DateTime'], axis=1, inplace=False)
rng = rng.set_index('DateTime')

# uretim1 = uretim.__gerceklesen(baslangic_tarihi='2016-01-01', bitis_tarihi='2021-04-30')
# uretim1['DateTime'] = pd.to_datetime(uretim1.Tarih) + uretim1.Saat.astype('timedelta64[h]')
# uretim1 = pd.merge(uretim1, rng, how="outer", on=["DateTime", "DateTime"])
# uretim1 = uretim1.set_index('DateTime')
# uretim1 = uretim1.drop('Tarih', axis = 1)
# uretim1 = uretim1.drop('Saat', axis = 1)
# uretim1 = pd.DataFrame(uretim1['Akarsu'])
# uretim1 = uretim1.replace(np.nan, 0)
# uretim1.sort_values(by='DateTime', ascending = True, inplace=True)

# uretim1 = uretim1.reset_index(drop=False)
# uretim1['year'] = pd.DatetimeIndex(uretim1['DateTime']).year
# uretim1['month'] = pd.DatetimeIndex(uretim1['DateTime']).month

# uretim1['Akarsu'].isnull().values.any()
# uretim1['Akarsu'].isnull().sum()
# uretim1['Akarsu'] = uretim1['Akarsu'].interpolate(method='pad', limit=6)
# uretim1['Akarsu'].iloc[2068] = uretim1['Akarsu'].iloc[2067]

# uretim1.to_excel('trfundamentalakarsu.xlsx')

#%% VERİ OKUMA

uretim1 = pd.read_excel('trfundamentalakarsu.xlsx',parse_dates=True)

uretim1['hour'] = pd.DatetimeIndex(uretim1['DateTime']).hour
uretim1['day'] = pd.DatetimeIndex(uretim1['DateTime']).day
uretim1['dayofyear'] = pd.DatetimeIndex(uretim1['DateTime']).dayofyear
uretim1['Hour_of_Year'] = (pd.DatetimeIndex(uretim1['DateTime']).dayofyear - 1) * 24 + pd.DatetimeIndex(uretim1['DateTime']).hour

#%% MODEL
minAkarsu = uretim1.groupby(['year','month'], as_index=False)['Akarsu'].min()
totalAkarsu = uretim1.groupby(['year','month'], as_index=False)['Akarsu'].sum()
maxAkarsu = uretim1.groupby(['year','month'], as_index=False)['Akarsu'].max()

minAkarsu1 = uretim1.groupby(['month'], as_index=False)['Akarsu'].min()
maxAkarsu1 = uretim1.groupby(['month'], as_index=False)['Akarsu'].max()

totalAkarsu['numdays'] = np.zeros(len(totalAkarsu))

for i in range(len(totalAkarsu)):
    year1 = totalAkarsu['year'][i]
    month1 = totalAkarsu['month'][i]
    total = number_of_days_in_month(year=year1, month=month1)
    totalAkarsu['numdays'][i] = total

AkarsuKuruluGuc = df['AKARSU']
totalAkarsu = pd.concat([totalAkarsu, AkarsuKuruluGuc], axis=1).reindex(maxAkarsu.index)

totalAkarsu['CF'] = totalAkarsu.Akarsu / (totalAkarsu.numdays * 24 * totalAkarsu.AKARSU)

minRuzgartotalAkarsu = totalAkarsu.groupby(['month'], as_index=False)['CF'].min()
maxRuzgartotalAkarsu = totalAkarsu.groupby(['month'], as_index=False)['CF'].max()

cf = np.zeros(len(maxRuzgartotalAkarsu))
cf = pd.DataFrame(cf)
cf = cf.set_axis(['CF'], axis=1, inplace=False)

for iteration2 in range(len(maxRuzgartotalAkarsu)):
    number = random.uniform(minRuzgartotalAkarsu['CF'][iteration2], maxRuzgartotalAkarsu['CF'][iteration2])
    cf['CF'][iteration2] = number

cf.to_excel('CFAKARSU.xlsx')

ax = np.arange(0, 12)
plt.figure(figsize=(8,5))
plt.fill_between(ax, minRuzgartotalAkarsu['CF'], maxRuzgartotalAkarsu['CF'], color="gray")
plt.plot(cf, color="red")
plt.title("AKARSU CAPACITY FACTOR")
plt.grid(True)
plt.legend(loc=0)
plt.show()

#%%########################################################

#############        SAATLİK MODEL        #################

###########################################################

minAkarsusaatlik = uretim1.groupby(['year','month','day','hour'], as_index=False)['Akarsu'].min()
totalAkarsusaatlik = uretim1.groupby(['year','month','day','hour'], as_index=False)['Akarsu'].sum()
maxAkarsusaatlik = uretim1.groupby(['year','month','day','hour'], as_index=False)['Akarsu'].max()

totalAkarsusaatlik['numdays'] = np.zeros(len(totalAkarsusaatlik))

for i in range(len(totalAkarsusaatlik)):
    year1 = totalAkarsusaatlik['year'][i]
    month1 = totalAkarsusaatlik['month'][i]
    total = number_of_days_in_month(year=year1, month=month1)
    totalAkarsusaatlik['numdays'][i] = total

#%% KURULU GÜÇ EKLEME

totalAkarsusaatlik['kurulu guc'] = np.zeros(len(totalAkarsusaatlik))
count = 0
for iteration3 in range(len(totalAkarsusaatlik)):
    if totalAkarsusaatlik['month'][iteration3] != df['Month'][count]:
        count = count + 1
    if totalAkarsusaatlik['month'][iteration3] == df['Month'][count]:
        totalAkarsusaatlik['kurulu guc'][iteration3] = df['AKARSU'][count]

totalAkarsusaatlik['CF'] = totalAkarsusaatlik['Akarsu'] / (totalAkarsusaatlik['kurulu guc'])

totalAkarsusaatlik = pd.concat([totalAkarsusaatlik, uretim1['Hour_of_Year']], axis=1).reindex(maxAkarsusaatlik.index)

minnn = totalAkarsusaatlik.groupby(['year','month'], as_index=False)['CF'].min()
meannn = totalAkarsusaatlik.groupby(['year','month'], as_index=False)['CF'].mean()
maxxx = totalAkarsusaatlik.groupby(['year','month'], as_index=False)['CF'].max()

mintotalAkarsusaatlik = totalAkarsusaatlik.groupby(['Hour_of_Year'], as_index=False)['CF'].min()
maxtotalAkarsusaatlik = totalAkarsusaatlik.groupby(['Hour_of_Year'], as_index=False)['CF'].max()
meantotalAkarsusaatlik = totalAkarsusaatlik.groupby(['Hour_of_Year'], as_index=False)['CF'].mean()

randomnumber = 50

random2 = np.zeros((len(maxtotalAkarsusaatlik),randomnumber))

for iteration2 in range(len(mintotalAkarsusaatlik)):
    number = np.random.uniform(mintotalAkarsusaatlik['CF'][iteration2], maxtotalAkarsusaatlik['CF'][iteration2],randomnumber).reshape(randomnumber,1)
    number = np.transpose(number)
    random2[iteration2] = number

# random2 = pd.DataFrame(random2)
# random2.to_excel('random2.xlsx')

saatlikcf = np.mean(random2, axis=1)
saatlikcf = pd.DataFrame(saatlikcf)
saatlikcf = saatlikcf.set_axis(['CF'], axis=1, inplace=False)
saatlikcf.to_excel('saatlikcfakarsu.xlsx')

x = np.arange(0, 8784)
ax = plt.figure().add_subplot(111)
ax.plot(saatlikcf, color="red", label="Random Values")
ax.fill_between(x, mintotalAkarsusaatlik['CF'], maxtotalAkarsusaatlik['CF'], color="gray")
ax.set_title("AKARSU CAPACITY FACTOR", fontsize=10)
ax.set_xlabel('Hours of Year', fontsize=12)
ax.set_ylabel('Capacity Factor', fontsize=12)
ax.legend(loc="best", prop={'size': 8})
ax.grid(True)


from django.conf.urls import url
from . import views

app_name = 'login'
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^check/$', views.check_login, name='check'),
    url(r'^out/$', views.logout_view, name='logout'),

    url(r'^cadastro/$', views.cadastro, name='cadastro'),
    url(r'^cadastro/salvar/$', views.salvar_usuario, name='salvar_usuario'),

    url(r'^alterar/$', views.alterar_login, name='alterar_login'),
    url(r'^alterar/salvar/$', views.salvar_login, name='salvar_login'),
]


"""
URL configuration for flight_rewards project.

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

from rest_framework import routers

from flight_rewards.flights.viewsets import (
    OriginAirportViewSet,
    DestinationAirportViewSet,
    ContactViewSet,
    AvailabilityNotificationViewSet,
    UserViewSet,
    SubscriptionPlanViewSet,
    FlightDepartureDatesViewSet,
    FlightViewSet,
    NotificationJobViewSet,
    AustralianFlightsViewSet
)
from flight_rewards.flights.views import upload_flights, upload_result

class OptionalSlashDefaultRouter(routers.DefaultRouter):

    def __init__(self):
        super().__init__()
        self.trailing_slash = '/?'

router = OptionalSlashDefaultRouter()
router.register(r'users', UserViewSet, basename='users')
router.register(r'origins', OriginAirportViewSet, basename='origins')
router.register(r'destinations', DestinationAirportViewSet, basename='destinations')
router.register(r'flights', FlightViewSet, basename='flights')
router.register(r'australia_flights', AustralianFlightsViewSet, basename='australia_flights')
router.register(r'flight-dates', FlightDepartureDatesViewSet, basename='flight-departure-dates')
router.register(r'contacts', ContactViewSet)
router.register(r'alerts', AvailabilityNotificationViewSet, basename='alerts')
router.register(r'alert_job', NotificationJobViewSet, basename='alert_job')
router.register(r'plans', SubscriptionPlanViewSet, basename='plans')

urlpatterns = [
    path('admin/', admin.site.urls),
    path('api/', include(router.urls)),
    path('api/', include('djoser.urls')),
    path('api/', include('djoser.urls.jwt')),
    path('api-auth/', include('rest_framework.urls')),
    path('flights/upload/', upload_flights, name='upload_flights'),
    path('flights/upload_result/', upload_result, name='upload_result'),
    path('stripe/', include('djstripe.urls', namespace='djstripe'))
]

if settings.DEBUG:
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)


from PIL import Image
import sys
import numpy as np

from .vid import VIDDataset
from mega_core.config import cfg

class VIDMEGADataset(VIDDataset):
    def __init__(self, image_set, data_dir, img_dir, anno_path, img_index, transforms, is_train=True):
        super(VIDMEGADataset, self).__init__(image_set, data_dir, img_dir, anno_path, img_index, transforms, is_train=is_train)
        if not self.is_train:
            self.start_index = []
            self.start_id = []
            if cfg.MODEL.VID.MEGA.GLOBAL.ENABLE:
                self.shuffled_index = {}
            for id, image_index in enumerate(self.image_set_index):
                frame_id = int(image_index.split("/")[-1])
                if frame_id == 0:
                    self.start_index.append(id)
                    if cfg.MODEL.VID.MEGA.GLOBAL.ENABLE:
                        shuffled_index = np.arange(self.frame_seg_len[id])
                        if cfg.MODEL.VID.MEGA.GLOBAL.SHUFFLE:
                            np.random.shuffle(shuffled_index)
                        self.shuffled_index[str(id)] = shuffled_index

                    self.start_id.append(id)
                else:
                    self.start_id.append(self.start_index[-1])

    def _get_train(self, idx):
        filename = self.image_set_index[idx]
        img = Image.open(self._img_dir % filename).convert("RGB")

        # if a video dataset
        img_refs_l = []
        img_refs_m = []
        img_refs_g = []
        if hasattr(self, "pattern"):
            offsets = np.random.choice(cfg.MODEL.VID.MEGA.MAX_OFFSET - cfg.MODEL.VID.MEGA.MIN_OFFSET + 1,
                                       cfg.MODEL.VID.MEGA.REF_NUM_LOCAL, replace=False) + cfg.MODEL.VID.MEGA.MIN_OFFSET
            for i in range(len(offsets)):
                ref_id = min(max(self.frame_seg_id[idx] + offsets[i], 0), self.frame_seg_len[idx] - 1)
                ref_filename = self.pattern[idx] % ref_id
                img_ref = Image.open(self._img_dir % ref_filename).convert("RGB")
                img_refs_l.append(img_ref)

            # memory frames
            if cfg.MODEL.VID.MEGA.MEMORY.ENABLE:
                ref_id_center = max(self.frame_seg_id[idx] - cfg.MODEL.VID.MEGA.ALL_FRAME_INTERVAL, 0)
                offsets = np.random.choice(cfg.MODEL.VID.MEGA.MAX_OFFSET - cfg.MODEL.VID.MEGA.MIN_OFFSET + 1,
                                           cfg.MODEL.VID.MEGA.REF_NUM_MEM, replace=False) + cfg.MODEL.VID.MEGA.MIN_OFFSET
                for i in range(len(offsets)):
                    ref_id = min(max(ref_id_center + offsets[i], 0), self.frame_seg_len[idx] - 1)
                    ref_filename = self.pattern[idx] % ref_id
                    img_ref = Image.open(self._img_dir % ref_filename).convert("RGB")
                    img_refs_m.append(img_ref)

            # global frames
            if cfg.MODEL.VID.MEGA.GLOBAL.ENABLE:
                ref_ids = np.random.choice(self.frame_seg_len[idx], cfg.MODEL.VID.MEGA.REF_NUM_GLOBAL, replace=False)
                for ref_id in ref_ids:
                    ref_filename = self.pattern[idx] % ref_id
                    img_ref = Image.open(self._img_dir % ref_filename).convert("RGB")
                    img_refs_g.append(img_ref)
        else:
            for i in range(cfg.MODEL.VID.MEGA.REF_NUM_LOCAL):
                img_refs_l.append(img.copy())
            if cfg.MODEL.VID.MEGA.MEMORY.ENABLE:
                for i in range(cfg.MODEL.VID.MEGA.REF_NUM_MEM):
                    img_refs_m.append(img.copy())
            if cfg.MODEL.VID.MEGA.GLOBAL.ENABLE:
                for i in range(cfg.MODEL.VID.MEGA.REF_NUM_GLOBAL):
                    img_refs_g.append(img.copy())

        target = self.get_groundtruth(idx)
        target = target.clip_to_image(remove_empty=True)

        if self.transforms is not None:
            img, target = self.transforms(img, target)
            for i in range(len(img_refs_l)):
                img_refs_l[i], _ = self.transforms(img_refs_l[i], None)
            for i in range(len(img_refs_m)):
                img_refs_m[i], _ = self.transforms(img_refs_m[i], None)
            for i in range(len(img_refs_g)):
                img_refs_g[i], _ = self.transforms(img_refs_g[i], None)

        images = {}
        images["cur"] = img
        images["ref_l"] = img_refs_l
        images["ref_m"] = img_refs_m
        images["ref_g"] = img_refs_g

        return images, target, idx

    def _get_test(self, idx):
        filename = self.image_set_index[idx]
        img = Image.open(self._img_dir % filename).convert("RGB")

        # give the current frame a category. 0 for start, 1 for normal
        frame_id = int(filename.split("/")[-1])
        frame_category = 0
        if frame_id != 0:
            frame_category = 1

        img_refs_l = []
        # reading other images of the queue (not necessary to be the last one, but last one here)
        ref_id = min(self.frame_seg_len[idx] - 1, frame_id + cfg.MODEL.VID.MEGA.MAX_OFFSET)
        ref_filename = self.pattern[idx] % ref_id
        img_ref = Image.open(self._img_dir % ref_filename).convert("RGB")
        img_refs_l.append(img_ref)

        img_refs_g = []
        if cfg.MODEL.VID.MEGA.GLOBAL.ENABLE:
            size = cfg.MODEL.VID.MEGA.GLOBAL.SIZE if frame_id == 0 else 1
            shuffled_index = self.shuffled_index[str(self.start_id[idx])]
            for id in range(size):
                filename = self.pattern[idx] % shuffled_index[
                    (idx - self.start_id[idx] + cfg.MODEL.VID.MEGA.GLOBAL.SIZE - id - 1) % self.frame_seg_len[idx]]
                img = Image.open(self._img_dir % filename).convert("RGB")
                img_refs_g.append(img)

        target = self.get_groundtruth(idx)
        target = target.clip_to_image(remove_empty=True)

        if self.transforms is not None:
            img, target = self.transforms(img, target)
            for i in range(len(img_refs_l)):
                img_refs_l[i], _ = self.transforms(img_refs_l[i], None)
            for i in range(len(img_refs_g)):
                img_refs_g[i], _ = self.transforms(img_refs_g[i], None)

        images = {}
        images["cur"] = img
        images["ref_l"] = img_refs_l
        images["ref_g"] = img_refs_g
        images["frame_category"] = frame_category
        images["seg_len"] = self.frame_seg_len[idx]
        images["pattern"] = self.pattern[idx]
        images["img_dir"] = self._img_dir
        images["transforms"] = self.transforms

        return images, target, idx


import cv2
import sys
import os
import glob
import numpy as np
from pyzbar.pyzbar import decode
import darknet_video

CFG = "./cfg/yolov3_bb.cfg"
WEIGHT = "./backup/bb/yolov3_bb_last.weights"
META = "./cfg/bb.data"

WIDTH = 640
HEIGHT = 480

#resize gray 59%
#gray 62%
#resize gray binary 6.9%
#gray binary 35%

def cut_barcode(img, detections, is_gray):
	scale = 0.1
	points = darknet_video.convert(detections)
	for point in points:
		x = point[0] - int(point[0] * scale)
		y = point[1] - int(point[1] * scale)
		w = point[2] + int(point[2] * scale)
		h = point[3] + int(point[3] * scale)

		cut_img = img[y:h, x:w]
		if is_gray:
			cut_img = cv2.cvtColor(cut_img, cv2.COLOR_BGR2GRAY)

		return cut_img

def read(img):
	try:
		result = decode(img)
		return True, result[0][0]
	except Exception as e:
		return False, -1

def write(fp, flag, num, file):
	txt = ""
	if flag:
		txt = "{}:{}".format(os.path.basename(file), num)
		fp.write(txt)
	else:
		txt = "{}:{}".format(os.path.basename(file), "failed")
		fp.write(txt)

	fp.write("\n")
	fp.flush()

#pyzbar only
def exe_py(folder):
	with open("result_py.txt", "a") as f:
		files = glob.glob(folder + os.sep + "*.png")
		count = 0
		for file in files:
			img = cv2.imread(file)
			flag, num = read(img)
			if flag:
				count = count + 1
			write(f, flag, num, file)

	per = count / len(files) * 100
	print("-"*100)
	print("{}/{}".format(count, len(files)))
	print("{}%".format(per))

#grayscale
def exe_py_gray(folder, is_denoising):
	with open("result_gray.txt", "a") as f:
		files = glob.glob(folder + os.sep + "*.png")
		count = 0
		for file in files:
			img = cv2.imread(file)
			gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

			if is_denoising:
				gray = cv2.fastNlMeansDenoising(gray, None, 10, templateWindowSize=7, searchWindowSize=21)

			flag, num = read(gray)
			if flag:
				count += 1

			write(f, flag, num, file)
			#gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
			#binary = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
			#	cv2.THRESH_BINARY, 11, 2)

	per = count / len(files) * 100
	print("-"*100)
	print("{}/{}".format(count, len(files)))
	print("{}%".format(per))

def exe_py_yolo(folder, is_gray, is_denoising):
	yolo = darknet_video.YOLO(CFG, WEIGHT, META)
	yolo.start_yolo()

	with open("result_yolo.txt", "a") as f:
		files = glob.glob(folder + os.sep + "*.png")
		count = 0
		for file in files:
			img = cv2.imread(file)

			detections, resize_img = yolo.run_yolo(img)
			if len(detections) > 0:
				barcode = cut_barcode(resize_img, detections, is_gray)
				if is_denoising:
					barcode = cv2.fastNlMeansDenoising(barcode, None, h=10, templateWindowSize=7, searchWindowSize=21)

				#cv2.imwrite("{}_y.png".format(count), barcode)
				flag, num = read(barcode)
				if flag:
					count += 1
				#	print("S")
				write(f, flag, num, file)
			else:
				#print("F")
				write(f, False, -1, file)

	per = count / len(files) * 100
	print("-"*100)
	print("{}/{}".format(count, len(files)))
	print("{}%".format(per))

def main():
	argv = sys.argv
	if len(argv) != 3:
		print("Usage: python {} folder type(1:pyonly 2:gray 3:gray&denoising, 4:yolo 5:yolo&gray 6:yolo&gray&denoising)".format(argv[0]))
		quit(-1)

	folder = argv[1]
	mode = int(argv[2])

	if mode == 1:
		exe_py(folder)
	elif mode == 2:
		exe_py_gray(folder, False)
	elif mode == 3:
		exe_py_gray(folder, True)
	elif mode == 4:
		exe_py_yolo(folder, False, False)
	elif mode == 5:
		exe_py_yolo(folder, True, False)
	elif mode == 6:
		exe_py_yolo(folder, True, True)

if __name__ == "__main__":
	main()


import os
import json
from pyrfc import Connection
from dotenv import load_dotenv, find_dotenv

def z_get_released_transports(age_in_days):
    env_file = find_dotenv(usecwd=True)
    load_dotenv(dotenv_path=env_file)

    connection_attributes = {
        'user'      : os.getenv('USER'),
        'passwd'    : os.getenv('PASSWORD'),
        'ashost'    : os.getenv('ASHOST'),
        'sysnr'     : os.getenv('SYSNR'),
        'client'    : os.getenv('CLIENT'),
    }

    if None in connection_attributes.values():
        return None

    with Connection(**connection_attributes) as connection:
        json_string = connection.call(
                "Z_GET_RELEASED_TRANSPORTS",
                AGE_IN_DAYS=age_in_days)['JSON_DATA']

    parsed_data = json.loads(json_string)
    return parsed_data


import argparse
import multiprocessing as mp
import os
import subprocess as sup
import sys
#-------------
"""
    Author: Gray
    Date:   2018-10-02
    Aim:    Run MinIONQC.r at -q 7 to 12
"""

def JOB(workTuple):
    Q, inputSummarySeq, workDir, Rscript, FileMarker = workTuple
    os.makedirs(os.path.join(workDir,"Q_"+str(Q)+"out"))

    if os.name == "nt":  #Windows
        EXE = "Rscript.exe"
    elif os.name == "posix": #Unix
        EXE = "Rscript"
    else:
        print("unknow OS")
        os._exit()

    cmd1 = [EXE, Rscript, "-i", inputSummarySeq,
    "-o", os.path.join(workDir,"Q_"+str(Q)+"out"), "-q", str(Q),
    "-m", FileMarker]
    sup.call(cmd1)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=sys.argv[0]+
                                     ": Run MinIONQC.R in Q7 to Q12")
    parser.add_argument('--script', dest="MainScript",
                    help="The MinIONQC script, for different modified version",
                    default=r"D:\ProjectWork\20181002_ONT_MinION_QC_modifying\MinIONQC_modByGray_20181002.r")
    parser.add_argument('-i', '--input', dest="INPUT", required=True,
                    help="input inputSummaryText from albacore")
    parser.add_argument('-m', '--marker', dest="Marker", required=True,
                    help="The marker as tag for yaml result sheets")
    parser.add_argument('-o', '--outdir', dest="OUTDIR", required=True,
                    help="output work dir, auto-create Q7 to Q12 folders")
    parser.add_argument("--core", dest="CORE", default=1)

    results = parser.parse_args()
    INPUT = results.INPUT
    WORKDIR = results.OUTDIR
    Rscript = results.MainScript
    FileMarker = results.Marker
    core = int(results.CORE)
    mp.freeze_support()
    worklist = [ (Q, INPUT, WORKDIR, Rscript, FileMarker) for Q in range(7,13)]
    pool = mp.Pool(core) #os.cpu_count()
    pool.map(JOB, worklist)


"""
https://flask-sqlalchemy.palletsprojects.com/en/3.0.x/quickstart/
"""

from flask import Flask, request, jsonify
from flask_sqlalchemy import SQLAlchemy

# Create the extension
db = SQLAlchemy()

# Create the app
app = Flask(__name__)

# Create a SQLite Database called data.db in the same directory
# relative to the app instance folder
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///data.db"

# Initialize the app with the extension
db.init_app(app)

# Define Models
class Drink(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), unique=True, nullable=False)
    description = db.Column(db.String(120))

    def __repr__(self):
        return f"{self.name} - {self.description}"

# Call SQLAlchemy.create_all() to create the table schema in the database
with app.app_context():
    db.create_all()

@app.route('/')
def index():
    return "Hello!"

@app.route("/drinks")
def get_drinks():
    drinks = Drink.query.all()
    output = []
    for drink in drinks:
        drink_data = {"name": drink.name, "description": drink.description}
        output.append(drink_data)

    return {"drinks": output}, 200

@app.route("/drinks/<id>")
def get_drink(id):
    drink = Drink.query.get_or_404(id)
    return {"name": drink.name, "description": drink.description}, 200

@app.route("/drinks", methods=["POST"])
def add_drink():
    drink = Drink(name=request.json["name"], description=request.json["description"])
    db.session.add(drink)
    db.session.commit()
    return {"id": drink.id}, 201

@app.route("/drinks/<id>", methods=["DELETE"])
def delete_drink(id):
    """ drink = Drink.query.get(id)
    if drink is None:
        return {"error": "Not Found"}, 404
    db.session.delete(drink)
    db.session.commit()
    return {"id": drink.id}, 200 """
    drink = Drink.query.get_or_404(id)
    # This way, we keep the 404 HTML Error page if the id does not exist
    #print(drink)

    # If the id does not exist in the DB, the drink variable will be 'None'
    # and it does not matter if it is passed on to db.session.delete(drink).
    # It will attempt to delete a None object, which does nothing.
    db.session.delete(drink)
    db.session.commit()
    return {"id": drink.id}, 200

if __name__ == "__main__":
    app.run(debug=True, host="127.0.0.1", port=81)


#!/usr/bin/env python3
import re
import sys
import os
import difflib
import fcntl
import time
import locale
import datetime
import os
from io import BytesIO
import subprocess
import urllib
from collections import deque
import base64
import argparse

from pprint import pprint
from get_mongo_client import get_mongo_client
import pytz
import yaml
from dateutil.parser import parse
import simplejson as json
import requests
from bs4 import BeautifulSoup
from PIL import Image
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
import numpy as np
from googleapiclient.discovery import build
from get_tweepy import *

def save_tweet(no_retweet=False, ids=None, screen_names=None):
    """Retweet tweets which have the hash_tag by REST API.

    Args:
        no_retweet:
            Save the tweet but does not retweet it if True.
        ids:
            Gets tweets of the ids not tag search results.
    """
    if ids:
        ts = api.statuses_lookup(ids)
    elif screen_names:
        ts = []
        for sn in screen_names:
            ts += [t for t in tweepy.Cursor(api.user_timeline,
                                           screen_name=sn,
                                           count=200,
                                           tweet_mode='extended').items(2000)]
        ts = [t for t in map(assign_text_to_full_text, ts) if is_right_tweet(t)]
    else:
        ts = [t for t in tweepy.Cursor(api.search,
                                       q='#' + setting['hash_tag'] + ' -RT',
                                       count=200,
                                       tweet_mode='extended').items(500)]
        ts = list(map(assign_text_to_full_text, ts))
    for t in reversed(ts):
        if is_right_tweet(t):
            record(t)
            store_image(t.id)
            if not no_retweet:
                retweet(t)

def assign_text_to_full_text(t):
    t.text = t.full_text
    t._json['text'] = t._json['full_text']
    return t

def is_right_tweet(t):
    """
    Check whether the tweet is to be recorded to database and retweeted.
    """
    return not has_id(t.id) and is_not_spam(t)

def record(t):
    """
    Record a tweet to database.
    """
    if not has_id(t.id):
        doc = make_doc(t)
        tweets.insert_one(doc)

def retweet(t):
    """
    Retweet the tweet.
    """
    try:
        res = t.retweet()
        id = res.id
    except tweepy.TweepError as e:
        if e.api_code == 327: # already retweeted
            id = t.id
        else:
            raise
    tweets.update_one({'_id': t.id},
                      {'$set': {
                          'meta.retweeted': True,
                          'meta.retweet_id': id,
                      }})

def make_doc(t):
    """
    Make MongoDB doc object from the tweet.
    """
    time = parse(t._json['created_at']).astimezone(pytz.timezone('Asia/Tokyo'))
    doc = {
        '_id': t.id,
        'meta': {
            'time': time,
            'date': get_date(time),
            'retweeted': False,
            'removed': False,
            'deny_collection': False,
            'deny_retweet': False,
            'exception': False,
            'labels': ['none'],
            'imgs': [],
        },
        'tweet': t._json,
    }
    return doc

def get_date(time):
    """
    Get tweeting date of the tweet.
    """
    threshold_time = (parse(setting['start_time']) - datetime.timedelta(minutes=30)).time()
    if time.time() < threshold_time:
        return time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) - datetime.timedelta(days=1)
    else:
        return time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)

def get_date_pripara_prpr(time):
    # first threshold - on wednesday
    if parse('wed').weekday() == time.weekday():
        if time.time() < parse('21:00').time():
            delta = 4
        else:
            delta = 0

    # second threshold - on saturday
    if parse('sat').weekday() == time.weekday():
        if time.time() < parse('21:00').time():
            delta = 3
        else:
            delta = 0

    # first span - from thu to fri
    if parse('wed').weekday() < time.weekday() < parse('sat').weekday():
        delta = time.weekday() - parse('wed').weekday()

    # second span - from sun to tue
    if parse('sat').weekday() < time.weekday():
        delta = time.weekday() - parse('sat').weekday()
    elif time.weekday() < parse('wed').weekday():
        delta = time.weekday() + 2

    date = time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
    date -= datetime.timedelta(days=delta)
    return date

def retweet_and_record(tweet=False, id=False, retweet=True, fetch=True, exception=False):
    '''
    Retweet and record the tweet.
    Return a tweet record for other functions' use.
    Otherwise return None to tell them that a fetch failed.
    '''
    # get tweet from twitter or database
    if id:
        if fetch:
            try:
                tweet = api.get_status(id=id)
                if has_id(id):
                    set_value(id, 'meta.removed', False)

            except TweepError as e:
                print('-' * 16)
                if e.api_code == 404:   # tweet was deleted
                    if has_id(id):
                        set_value(id, 'meta.removed', 'deleted')
                        print('404 Not found and marked "deleted":', id)
                        print_tweet(id)
                        return None
                    else:
                        print('404 Not found and not in database:', id)
                elif e.api_code == 403:   # accound has been locked
                    if has_id(id):
                        set_value(id, 'meta.removed','locked')
                        print('403 Tweet has been locked and marked "locked":', id)
                        print_tweet(id)
                        return None
                    else:
                        print('403 Tweet has been locked and not in database:', id)
                        return None
                elif e.api_code == 429: # API remaining = 0
                    print('api not remaining')
                    return get_tweets('_id', id)[0]
                else:
                    print('* Unknown error:', id)
                    print(e)
                    if tweet:
                        pprint(tweet)
                    return None
    else:
        id = tweet['id']

    if not has_id(id):
        new = True
        if tweet:
            tweet = {'tweet': tweet}
    else:
        new = False
        if tweet:
            tweets.update_one({'_id': id}, {'$set': {'tweet': tweet}}, True)
        tweet = get_tweets('_id', id)[0]

    # excluding filters
    if is_not_spam(tweet, new=new) or exception:

        with open(setting['ignores']) as f:
            fcntl.flock(f, fcntl.LOCK_SH)
            ignores = yaml.load(f)

    return tweet

def including_hash_tag(t):
    """
    Check if the tweet includes the hashtag.
    """
    if 'hashtags' in t.entities: # tweet has hashtags data
        if any(filter(lambda tag: tag['text'] == setting['hash_tag'], t.entities['hashtags'])):
            return True
    print('no including hash_tag')
    return False

def is_not_spam(t):
    """
    Check the tweet is not a spam.
    """

    spam = ''
    # if ignore_id tweet
    if t.id in ignores['ignore_id']:
        spam = 'Hit ignore_id:'
    # if ignore_url
    elif t.entities['urls'] and any([ignore_url in url['expanded_url']
                                                     for ignore_url in ignores['ignore_url']
                                                     for url in t.entities['urls']]):
        spam = 'Hit ignore_url:'
    # if official retweet
    elif t.entities['user_mentions']:
        spam = 'Including user mentions:'
    # if mention
    elif 'retweeted_status' in t.entities:
        spam = 'Official retweet:'
    # if including hash_tag which similar to any of the hash_tags
    elif not including_hash_tag(t):
        spam = 'Not including any hash_tag:'
    # if including image
    elif 'http' not in t.text:
        spam = 'Not including http:'
        print(t.text)
    # if ignore_user
    elif t.user.id in ignores['ignore_user'] or t.user.screen_name in ignores['ignore_user']:
        spam = 'Hit ignore_user "{}":'.format(t.user.screen_name)
    # not in white list users
    elif (t.user.screen_name not in ignores['white_user']) \
         and (t.user.id not in  ignores['white_user']):
        # if recently created account
        if t.user.created_at > datetime.datetime.now() - datetime.timedelta(weeks=1):
            spam = 'Hit too recently created:'
        # if unsafe images
        # elif tweets.find({'tweet.user.id': t.user.id}).count() == 0 \
        #      and is_unsafe_image(t):
        #     add_ignore_users(t.user.id)
        #     cancel_user_retweet(t.user.id)
        #     spam = 'Hit unsafe image:'
    # if including ignore_word
    else:
        for word in ignores['ignore_word']:
            if re.search(word, t.text):
                spam += 'Hit ignore_word "{}":'.format(word)
    if spam:
        print('[is_not_spam()] hit spam by reason: {reason}'.format(reason=spam))
        print_tweet(t)
        return False
    else:
        return True

def check_new_ignore_user_list():
    """
    Check new ignore users and add them ignore list.
    """
    ls = api.get_list(owner_screen_name=setting['rt_account'], slug='list')
    user_ids = get_list_members(list_id=ls.id)
    user_ids = add_ignore_users(user_ids)
    cancel_user_retweet(user_ids)

def get_list_members(list_id):
    """
    Get ids of members in `list_id` list.
    """
    ids = [u.id for u in tweepy.Cursor(api.list_members, list_id=list_id, count=100).items()]
    return ids

def add_ignore_users(user_ids):
    """
    Add `user_id`'s user to ignore list.
    """
    if not type(user_ids) in (set, list):
        user_ids = [user_ids]

    with open(setting['ignores'], 'r+') as f:
        # Read file
        fcntl.flock(f, fcntl.LOCK_SH)
        ignores = yaml.load(f)

        # Add a user's screen_name and id
        new_ignore_user_ids = []
        for user_id in user_ids:
            if not user_id in ignores['ignore_user']:
                user = api.get_user(id=user_id)
                ignores['ignore_user'].append(user.screen_name)
                ignores['ignore_user'].append(user_id)
                new_ignore_user_ids.append(user_id)

        # Write file
        f.seek(0)
        f.truncate()
        yaml.dump(ignores, f, allow_unicode=True)

    # Return new added users
    return new_ignore_user_ids

def cancel_user_retweet(user_ids):
    """
    Destroy all the past retweets by the users.
    """
    if not type(user_ids) in [set, list]:
        user_ids = [user_ids]
    for user_id in user_ids:
        cancel_tweets = tweets.find({'tweet.user.id': user_id})
        for cancel_tweet in cancel_tweets:
            try:
                t = api.get_status(id=cancel_tweet['_id'])
                if t.retweeted and 'retweet_id' in cancel_tweet['meta']:
                    api.destroy_status(id=cancel_tweet['meta']['retweet_id'])
                    tweets.remove({'_id': cancel_tweet['_id']})
            except tweepy.TweepError as e:
                # Tweet removed
                if e.api_code == 404:
                    tweets.remove({'_id': cancel_tweet['_id']})
                else:
                    raise

def record_user(screen_name):
    """
    Record all the tweets of the user.
    """
    ts = api.user_timeline(screen_name=screen_name, count=100, tweet_mode='extended')
    ts = map(assign_text_to_full_text, ts)
    for t in ts:
        save_tweet(t, retweet=False)

def is_unsafe_image(t):
    """
    Detect whether the tweet includes unsafe images by Google Cloud Vision API.
    """
    print_tweet(t)

    # Get image url from tweet

    # The tweet has official images
    if getattr(t, 'extended_entities', False) and 'media' in t.extended_entities:
        urls = [media['media_url_https'] for media in t.extended_entities['media']]
    else:
        # if not images, pass the check
        return False

    # Request for Google Cloud Vision API
    for url in urls:
        r = requests.get(url)
        image = base64.b64encode(r.content).decode('UTF-8')
        body = {
            "requests": [
                {
                    'image': {
                        'content': image,
                    },
                    "features": [
                        {
                            "type": "SAFE_SEARCH_DETECTION",
                        },
                    ]
                },
            ]
        }
        res = vision.images().annotate(body=body).execute()

        # Chack unsafe possibility
        annotation = res['responses'][0]['safeSearchAnnotation']
        annotation.pop('spoof')
        print(annotation)
        unsafe_possible = any([possibility in ['LIKELY', 'VERY_LIKELY']
                               for possibility in annotation.values()])
        if unsafe_possible:
            print('Found unsafe image:')
            return True

    return False

def store_image(id):
    doc = tweets.find_one(id)
    tweet = doc['tweet']
    urls = []
    if 'extended_entities' in tweet \
       and 'media' in tweet['extended_entities']:
        # if tweet has official image
        for url in tweet['extended_entities']['media']:
            urls.append((url['media_url_https'] + ':orig', url['expanded_url']))
    elif 'media' in tweet['entities']:
        # if tweet has official image
        for url in tweet['entities']['media']:
            urls.append((url['media_url_https'] + ':orig', url['expanded_url']))
    else:
        for url in tweet['entities']['urls']:
            urls.append(url['expanded_url'])

    headers = {}
    imgs = []
    for url in urls:
        if any(['twimg.com' in x for x in url]):
            (img_url, url) = url
        elif 'twitpic.com' in url:
            soup = BeautifulSoup(requests.get(url).text, 'lxml')
            img = soup.select('#media-main img')
            if not img:
                img_url = False
            else:
                img_url = img[0]['src']

        elif 'photozou.jp' in url:
            soup = BeautifulSoup(requests.get(url).text, 'lxml')
            img_url = soup(itemprop='image')[0]['src']
        elif 'p.twipple.jp' in url:
            img_url = url.replace('p.twipple.jp/', 'p.twpl.jp/show/orig/')
        elif 'yfrog.com' in url:
            url = url.replace('yfrog.com', 'twitter.yfrog.com')
            r = requests.get(url)
            soup = BeautifulSoup(r.text, 'lxml')
            url = soup(id='continue-link')[0].a['href']
            r = requests.get(url)
            soup = BeautifulSoup(r.text, 'lxml')
            img_url = soup.select('.main-image a')[0]['href']
        elif 'pixiv' in url:
            headers = {'referer': 'http://www.pixiv.net/'}
            soup = BeautifulSoup(requests.get(url).text, 'lxml')
            img_url = soup('img')[1]['src']
        elif 'togetter.com' in url:
            img_url  = False
        elif re.search(r'((jpeg)|(jpg)|(png)|(gif))(.*)$', url):
            img_url = url
        elif re.search(r'twitter.com/.+?/status/.+?/photo', url):
            # check if the photo is animated gif
            soup = BeautifulSoup(requests.get(url).text, 'lxml')
            img_url = soup(class_='animated-gif-thumbnail')[0]['src']
            if not img_url:
                img_url = False
        else:
            img_url = False

        imgs.append({'url': url, 'img_url': img_url})

    # set imgs after save all the imgs loop
    tweets.update_one({'_id': id}, {'$set': {'meta.imgs': imgs}})
    return imgs

def remove_ignore_tweets():
    with open(setting['ignores']) as f:
        ignores = yaml.load(f)
    for user in ignores['ignore_user']:
        p = re.compile('.*' + str(user) +'.*')
        for i in tweets.find({'$or': [{'tweet.user.screen_name': {'$regex': p}}, {'tweet.user.id': {'$regex': p}}]}):
            print(i['tweet']['user']['screen_name'])
        tweets.delete_one({'$or': [{'tweet.user.screen_name': p}, {'tweet.user.id': p}]}, multi=True)

def make_symlinks_to_img_dir():
    for tweet in get_tweets():
        src_dir = setting['img_dir'] + t['tweet']['user']['id_str']
        dst_dir = setting['img_dir'] + '-' + tweet['tweet']['user']['screen_name']
        if not os.path.islink(dst_dir) and os.path.isdir(src_dir):
            os.symlink(os.getcwd() + '/' + src_dir, os.getcwd() + '/' + dst_dir)

# database
def set_value(id, key, val):
    tweets.update_one({'_id': id}, {'$set': {key: val}}, True)

def has_id(id):
    return bool(tweets.find({'_id': id}).count())

def get_tweets(key=None, value=None, sort=None):
    # init
    if not sort:
        sort = '_id'
    if not key and not value:
        return tweets.find().sort(sort)
    elif key and value:
        return tweets.find({key: value}).sort(sort)
    else:
        raise Error('Error: Give get_tweets key-value pair')

def print_tweet(t):
    if type(t) == dict:
        t = t['tweet']
    elif type(t) == tweepy.Status:
        t = t._json
    else:
        raise TypeError

    # Print tweet
    tweet_url = 'https://twitter.com/{name}/status/{id}'.format(
        name=t['user']['screen_name'],
        id=t['id'],
    )

    print('-' * 8)
    print(t['created_at'], '/ ♡ {} ↻ {} / {}'.format(
        t['favorite_count'],
        t['retweet_count'],
        tweet_url,
    ))
    print('{}(@{})'.format(t['user']['name'], t['user']['screen_name']))
    print(t['text'])

def update_date(daily=False, date=''):
    '''
    Update tweets statuses from new to old until api remaining become zero.

    param: daily - if True, append date tweets to the first of que.
    '''

    # initialize que file
    if not os.path.exists(setting['update_date_que']):
        with open(setting['update_date_que'], 'w') as f:
            yaml.dump(deque([]), f)

    # add daily tweets to the left of ques
    if daily:
        ques = deque([])
        for i in tweets.find({
                'meta.date': get_date(),
                'meta.removed': False,
                'meta.deny_collection': False,
        }).sort([('_id', -1)]):
            ques.appendleft(i['_id'])
    # add the date tweets to the left of ques
    elif date:
        ques = deque([])
        for i in tweets.find({
                'meta.date': date,
                'meta.removed': False,
                'meta.deny_collection': False,
        }).sort([('_id', -1)]):
            ques.appendleft(i['_id'])
    # load que file
    else:
        with open(setting['update_date_que']) as f:
            ques = yaml.load(f)

    # load tweets if there is no que
    if not ques:
        for i in tweets.find({
                'meta.removed': False,
                'meta.deny_collection': False,
        }).sort([('_id', -1)]):
            ques.append(i['_id'])

    # update tweets while api remaining
    api_remaining = get_show_status_remaining()
    fetch_count = api_remaining[0] - 5
    print('api remaining:', api_remaining[0])
    print('api reset time:', api_remaining[1])
    if fetch_count:
        for i in range(min(fetch_count, len(ques))):
            id = ques.popleft()
            tweet = tweets.find_one({'_id': id})
            retweet_and_record(id=id, retweet=False)
            print_tweet_summary(i, tweet)

        # save que
        with open(setting['update_date_que'], 'w') as f:
            yaml.dump(ques, f)

def print_tweet_summary(i, tweet):
    print('-' * 16)
    print('#{}'.format(i), tweet['meta']['date'], 'https://twitter.com/{}/statuses/{}'.format(tweet['tweet']['user']['screen_name'], tweet['_id']))
    print('{} (@{}) [{}/{}]'.format(tweet['tweet']['user']['name'], tweet['tweet']['user']['screen_name'], tweet['tweet']['favorite_count'], tweet['tweet']['retweet_count']))
    print(tweet['tweet']['text'])

# api remainings
def get_search_remaining():
    api = t.get_application_rate_limit_status()['resources']['search']['/search/tweets']
    api_remaining = api['remaining']
    api_reset = datetime.datetime.fromtimestamp(api['reset'])
    return (api_remaining, api_reset)

def get_show_status_remaining():
    api = t.get_application_rate_limit_status()['resources']['statuses']['/statuses/show/:id']
    api_remaining = api['remaining']
    api_reset = datetime.datetime.fromtimestamp(api['reset'])
    return (api_remaining, api_reset)

def sleep_until_api_reset():
    remaining, reset = get_show_status_remaining()
    if remaining == 0:
        print('There is no api remaining. Sleep until', reset)
        now = datetime.datetime.now()
        time.sleep((reset - now).seconds + 10)

def save_themes_yaml():
    with open(setting['themes']) as f:
        fcntl.flock(f, fcntl.LOCK_SH)
        themes_yaml = yaml.load(f)
        print(themes_yaml)
    for theme in themes_yaml:
        themes.update_one({'date': theme['date']}, {'$set': theme}, True)

def write_themes_yaml():
    with open(setting['themes'], 'w') as f:
        fcntl.flock(f, fcntl.LOCK_EX)
        yaml.dump([i for i in themes.find().sort('date')], f, allow_unicode=True)

def update_themes():
    '''
    Get a new theme and togetter url from the official account's tweets,
    and count tweets of the date in the database.
    '''
    ts = api.user_timeline(screen_name=setting['account'], count=100, tweet_mode='extended')
    ts = map(assign_text_to_full_text, reversed(ts))

    # update theme and togetter
    for t in ts:
        date = get_date(parse(t._json['created_at']).astimezone())
        print_tweet(t)

        # replace newline with space for theme matching
        t.text = t.text.replace('\n', ' ')

        # insert new theme of the day
        if not themes.find({'date': date}).count():
            match = re.findall(setting['theme_regex'], t.text)
            print('Date:', date)
            print('Found a new theme:', match)

            if match:
                theme = {}
                theme_name = ' / '.join(match)
                theme['theme'] = theme_name

                # if there has already exists the same theme, copy theme_en and category
                same_theme = themes.find({'theme': theme_name}).sort([('meta.date', -1)])

                # copy theme
                if same_theme.count():
                    same_theme = same_theme[0]
                    theme['category'] = same_theme['category']
                    theme['theme_en'] = same_theme['theme_en']

                # otherwise, initialize
                else:
                    theme['category'] = ['uncategorized']
                    theme['theme_en'] = ''

                # for request theme
                match = re.search(r'\(.*リクエスト.*\)', t.text)
                if match:
                    theme['category'].append('request')
                    theme['theme'] = re.sub('\(.*リクエスト.*\)', '', theme['theme'])

                # for season theme
                match = re.search(r'\(.*季節.*\)', t.text)
                if match:
                    theme['category'] = ['season']
                    theme['theme'] = re.sub('\(.*季節.*\)', '', theme['theme'])

                # for cloth theme
                match = re.search(r'\(.*衣装.*\)', t.text)
                if match:
                    theme['category'] = ['clothes']
                    theme['theme'] = re.sub('\(.*衣装.*\)', '', theme['theme'])

                # for episode theme
                match = re.search(r'\(.*エピソード.*\)', t.text)
                if match:
                    theme['category'] = ['episode']
                    theme['theme'] = re.sub('\(.*エピソード.*\)', '', theme['theme'])

                # for episode theme
                match = re.search(r'\(.*アンケート.*\)', t.text)
                if match:
                    theme['category'] = ['questionnaire']
                    theme['theme'] = re.sub('\(.*アンケート.*\)', '', theme['theme'])

                # update database
                themes.update_one({'date': date}, {'$set': theme}, True)

                # retweet a official theme tweet
                #t.retweet()

        # update togetter url
        if setting['togetter']:
            non_togetter_tweets = themes.find({'date': date, 'togetter': {'$exists': False}})
            if non_togetter_tweets.count():
                if t.entities['urls'] and 'togetter.com/li/' in t.entities['urls'][0]['expanded_url']:
                    togetter = t.entities['urls'][0]['expanded_url']
                    themes.update_one({'date': date}, {'$set': {'togetter': togetter}}, True)

    # count event number and work number
    for num, theme in enumerate(themes.find().sort('date')):
        work_num = tweets.find({
            'meta.date': theme['date'],
            'meta.removed': False,
            'meta.deny_collection': False
        }).count()

        # Reset count for @precure_1draw_2
        if args.genre == 'precure' and theme['date'] and \
           theme['date'] >= parse('2017/1/14'):
            num -= 1000

        themes.update_one({'date': theme['date']},
                          {'$set': {'num': num, 'work_num': work_num}},
                          upsert=True)

    # count user number
    for i, date in enumerate(reversed(themes.distinct('date'))):
        if i > 20:
            break
        if not date:
            themes.update_one({'date': date}, {'$set': {'user_num': 0}})
        else:
            user_num = len(tweets.find({
                'meta.date': {'$lte': date},
                'meta.deny_collection': False,
                'meta.removed': False
            }).distinct('tweet.user.id'))
            themes.update_one({'date': date}, {'$set': {'user_num': user_num}}, True)

    write_themes_yaml()

def update_users():
    user_screen_names = tweets.find({
        'meta.deny_collection': False,
        'meta.removed': False,
        'tweet.user.screen_name': {
            '$not': re.compile(r'^{}$'.format(setting['account'][0])),
        },
    }).distinct('tweet.user.screen_name')
    users.delete_many({})
    for user_screen_name in user_screen_names:
        # get the latest tweet
        tweet = tweets.find(
            {'tweet.user.screen_name': user_screen_name},
            {'_id': {'$slice': -1}}
        )[0]
        num = tweets.find({'tweet.user.screen_name': user_screen_name, 'meta.removed': False}).count()
        cls = num // 10 * 10
        users.update_one(
            {'id': user_screen_name },
            {'$set': {
                'screen_name': user_screen_name,
                'num': num,
                'class': cls,
                'user': tweet['tweet']['user']}
            },
            upsert=True,
        )

def update_infos():
    with open(setting['info']) as f:
        infos_yaml = yaml.load(f)
    for id, info in enumerate(reversed(infos_yaml)):
        info_dict = {}
        info_dict['id'] = id
        for k, v in info.items():
            info_dict[k] = v
        print(info_dict)
        infos.update_one({'id': id}, info_dict, True)

def get_user_work_number(id):
    '''Return the number of the id tweet work of the user.'''
    screen_name = get_tweets('_id', id)[0]['tweet']['user']['screen_name']
    tweets = get_tweets('tweet.user.screen_name', screen_name)
    for num, tweet in enumerate(tweets):
        if tweet['_id'] == id:
            return num + 1

def update_labels(id, force=False):
    '''Update labels of the database which is used at each tweet's header.'''
    tweet = get_tweets('_id', id)[0]
    lucky_nums = list(range(10, 1001, 10))
    labels = []

    if 'labels' in tweet['meta'] and not force:
        return

    # lucky number
    num = get_user_work_number(id)
    if num == 1:
        labels.append('初参加')
    else:
        for lucky_num in lucky_nums:
            if num == lucky_num:
                labels.append(str(lucky_num))

    # gif
    if 'imgs' in tweet['meta']:
        if '/tweet_video_thumb/' in str(tweet['meta']['imgs'][0]['img_url']): # str() for bool value
            labels.append('GIF')

    if not labels:
        labels = ['none']

    print('set labels:',labels)
    set_value(id, 'meta.labels', labels)

def update_labels_all():
    screen_names = tweets.distinct('tweet.user.screen_name')
    lucky_nums = list(range(10, 1001, 10))

    for sn in screen_names:
        print('-' * 16)
        print('Update labels of user:', sn)
        user_tweets = tweets.find({'tweet.user.screen_name': sn, 'meta.removed': False}).sort('_id')

        for num, tweet in enumerate(user_tweets, 1):
            labels = []
            print_tweet_summary(num, tweet)

            # Add lucky number
            if num == 1:
                labels.append('初参加')
            else:
                if num in lucky_nums:
                    labels.append(str(num))

            # Add gif label
            if not 'imgs' in tweet['meta']:
                retweet_and_record(id=tweet['_id'], retweet=False)
                print('not imgs')
            imgs = tweet['meta']['imgs']
            if imgs and '/tweet_video_thumb/' in str(imgs[0]['img_url']): # str() for bool value
                print(tweet)
                labels.append('GIF')

            if not labels:
                labels = ['none']

            print(tweet['_id'], 'Set labels:', labels)
            set_value(tweet['_id'], 'meta.labels', labels)

def make_chart():
    def col_list(key):
        return [i[key] for i in themes.find().sort('num') if i['date']]

    days = col_list('num')
    nums = col_list('work_num')
    user_nums = col_list('user_num')

    # ax1: works num, ax2: user num
    fig, ax = plt.subplots()

    ax1 = plt.axes()
    ax2 = ax1.twinx()

    ax1.plot(days, user_nums, '-', color='lightsteelblue', linewidth=1)
    ax2.plot(days, nums, '-', color='palevioletred', linewidth=1)

    fig.autofmt_xdate()

    # set limit
    ax1.set_xlim(0, max(days) + 10)
    ax1.set_ylim(0, max(user_nums) + 30)
    ax2.set_ylim(0, max(nums) + 15)

    # set locator
    ax1.xaxis.set_major_locator(plt.MultipleLocator(200))
    ax1.yaxis.set_major_locator(plt.MultipleLocator(200))
    ax2.xaxis.set_major_locator(plt.MultipleLocator(100))
    ax2.yaxis.set_major_locator(plt.MultipleLocator(25))

    # set grid
    ax2.grid(True)

    # label
    fp = FontProperties(fname='Hiragino Sans GB W3.otf')
    ax1.set_xlabel('回数', fontproperties=fp)
    ax1.set_ylabel('累計参加者数', fontproperties=fp)
    ax2.set_ylabel('作品数/回', fontproperties=fp)

    # legend
    p1 = plt.Rectangle((0, 0), 1, 1, fc="lightsteelblue", ec='lightsteelblue')
    p2 = plt.Rectangle((0, 0), 1, 1, fc="palevioletred", ec='palevioletred')
    ax1.legend([p1, p2], ['累計参加者数', '作品数'], loc='upper left', prop=fp, )

    #plt.title('作品数の変化', fontproperties=fp)
    plt.savefig(setting['static_dirs'] + 'chart.svg')

    # en
    ax1.set_xlabel('#', fontproperties=fp)
    ax1.set_ylabel('Total perticipants')
    ax2.set_ylabel('Works')
    ax1.legend([p1, p2], ['Total perticipants', 'Works'], loc='upper left')
    plt.savefig(setting['static_dirs'] + 'chart-en.svg')

def generate_rank_html():
    '''Generate user rank pages.'''
    with open(setting['rank_users']) as f:
        fcntl.flock(f, fcntl.LOCK_SH)
        screen_names = yaml.load(f)

    for screen_name in screen_names:
        favs = {}
        imgs = []
        for theme in themes.find().sort([('date', -1)]):
            user_tweet = tweets.find_one({'meta.date': theme['date'], 'tweet.user.screen_name': screen_name})
            if not user_tweet:
                continue
            favs[theme['date']] = [fav_plus_rt(tweet) for tweet in tweets.find({'meta.date': theme['date']})]
            user_fav = fav_plus_rt(user_tweet)

            fig, ax = plt.subplots()
            n, bins, patches = plt.hist(favs[theme['date']], color='skyblue', bins=50)
            idx = (np.abs(bins - user_fav)).argmin()
            idx = min(idx, len(patches)-1)
            #print(idx, patches)
            if patches[idx].get_height():
                patches[idx].set_facecolor('palevioletred')
            elif patches[idx+1].get_height():
                patches[idx+1].set_facecolor('palevioletred')
            else:
                patches[idx-1].set_facecolor('palevioletred')
            fp = FontProperties(fname='Hiragino Sans GB W3.otf')
            ax.set_xlabel('Fav+RT', fontproperties=fp)
            ax.set_ylabel('人数', fontproperties=fp)
            save_dir = setting['img_dir'] + user_tweet['tweet']['user']['id_str']
            if not os.path.exists(save_dir):
                os.mkdir(save_dir)
            filename = str(theme['date'].date()) + '.svg'
            plt.savefig(save_dir +  '/' + filename)
            plt.close()

            total = len(favs[theme['date']])
            rank = sorted(favs[theme['date']], reverse=True).index(user_fav) + 1
            percent = int((rank / total) * 100)

            text = 'Fav+RT: {}<br>Rank: {} / {} ({}%)'.format(user_fav, rank, total, percent)

def fav_plus_rt(tweet):
    fav = tweet['tweet']['favorite_count']
    rt = tweet['tweet']['retweet_count']
    if fav + rt > 150:
        return 150
    else:
        return fav + rt

def show_status(id):
    if has_id(id):
        print_tweet(tweets.find_one(id)['tweet'])
    else:
        t = api.get_status(id=id, tweet_mode='extended')
        t = assign_text_to_full_text(t)
        print_tweet(t)

def init(_genre):
    # load setting file
    global genre
    global setting
    global ignores
    global api
    global vision
    global tweets
    global themes
    global users
    global infos

    genre = _genre
    with open('settings.yaml') as f:
        fcntl.flock(f, fcntl.LOCK_SH)
        setting = yaml.load(f)[genre]

    with open(setting['ignores']) as f:
        fcntl.flock(f, fcntl.LOCK_SH)
        ignores = yaml.load(f)

    # prepare twitter object
    api = get_api(setting['rt_account'])

    #  prepare Google Cloud Vision API
    with open('.google-api-key') as f:
        key = f.read().strip()
    vision = build('vision', 'v1', developerKey=key)

    tweets = get_mongo_client()[genre + '_1draw_collections']['tweets']
    themes = get_mongo_client()[genre + '_1draw_collections']['themes']
    users = get_mongo_client()[genre + '_1draw_collections']['users']
    infos = get_mongo_client()[genre + '_1draw_collections']['infos']

# Define global variable
genre = None
setting = None
ignores = None
api = None
vision = None
tweets = None
themes = None
users = None
infos = None

if __name__ == '__main__':
    command_choices = [
        'update_themes',
        'save_themes_yaml',
        'save_tweet',
        'update_users',
        'make_chart',
        'generate_rank_html',
        'check_new_ignore_user_list',
        'update_labels_all',
    ]
    parser = argparse.ArgumentParser()
    parser.add_argument('genre')
    parser.add_argument('command', choices=command_choices)
    parser.add_argument('--ids', nargs='+')
    parser.add_argument('--screen_names', nargs='+')
    parser.add_argument('--no-retweet', action='store_true')
    args = parser.parse_args()

    init(args.genre)

    if args.command == 'update_themes':
        update_themes()
    elif args.command == 'save_themes_yaml':
        save_themes_yaml()
    elif args.command == 'save_tweet':
        if args.ids:
            ids = [id.split('/')[-1] for id in args.ids]
            save_tweet(ids=ids)
        elif args.screen_names:
            save_tweet(screen_names=args.screen_names)
        else:
            save_tweet(no_retweet=args.no_retweet)
    elif args.command == 'update_users':
        update_users()
    elif args.command == 'update_labels_all':
        update_labels_all()
    elif args.command == 'make_chart':
        make_chart()
    elif args.command == 'generate_rank_html':
        generate_rank_html()
    elif args.command == 'check_new_ignore_user_list':
        check_new_ignore_user_list()

else:
    init('precure')


# compare the impedance of a metamaterial layer when calculated from S11
# and directly from the simulation
import numpy as np
from matplotlib import pyplot as plt

data = np.loadtxt("slab_0.4_3.35.txt", delimiter=",") # 0.4 mm FR4 file with increasing meshing
#data = np.loadtxt("slab_4_2.25.txt", delimiter=",") # 4 mm FR4 file

f = data[:,0] # in GHz
D = 76.93*1e-3 # 0.4 mm fr4 thickness
#D = (77.7258-4.5)*1e-3 # 4 mm fr4 thickness
S11 = (data[:,1] + 1j*data[:,2])*np.exp(4j*np.pi*f/3e8*D)
Z = (data[:,5] + 1j*data[:,6])#*np.exp(4j*np.pi*f/3e8*D)
epsFR4 = 4.4 - 1j*0.048957/(2*np.pi*f*8.8542e-12)
lz = 4e-4 # 0.4 mm FR4
#lz = 4e-3 # 4 mm FR4
Z0 = 376.73
Zslab = 1j*Z0/np.sqrt(epsFR4)*np.tan(2*np.pi*f/3e8*np.sqrt(epsFR4)*lz)

plt.plot(f/1e9, np.real((1+S11)/(1-S11))*Z0, "b-", label="Re(Z) from S, 76.9")
plt.plot(f/1e9, np.imag((1+S11)/(1-S11))*Z0, "r-", label="Im(Z) from S, 76.9")
plt.plot(f/1e9, np.real(Z), "bo", markersize=2,label="Re(Z), 76.83")
plt.plot(f/1e9, np.imag(Z), "ro", markersize=2,label="Im(Z), 76.83")
plt.plot(f/1e9, np.real(Zslab), "c-", label="Re(Z) analytic")
plt.plot(f/1e9, np.imag(Zslab), "m-", label="Im(Z) analytic")

plt.legend()
plt.xlim([2,8])
plt.ylim([-2,60])
plt.show()


import requests
from bs4 import BeautifulSoup
import pandas as pd
import re, time
from fake_useragent import UserAgent
import random
from crawler.craw_common import  *

class Sinyi():
    def __init__(self):
        self.name = 'sinyi'
        self.source = '信義房屋'
        self.user_agent = UserAgent()
        self.logger = WriteLogger().getLoggers()
        self.all_city_dict = {
            "A": "臺北市",
            "B": "臺中市",
            "C": "基隆市",
            "D": "臺南市",
            "E": "高雄市",
            "F": "新北市",
            "G": "宜蘭縣",
            "H": "桃園市",
            "I": "嘉義市",
            "J": "新竹縣",
            "K": "苗栗縣",
            "M": "南投縣",
            "N": "彰化縣",
            "O": "新竹市",
            "P": "雲林縣",
            "Q": "嘉義縣",
            "T": "屏東縣",
            "U": "花蓮縣",
            "V": "臺東縣",
            "X": "澎湖縣",
            "W": "金門縣",
            "Z": "連江縣"
            }

    def get_items(self, city_code):
        city = all_city_dict[city_code]
        house_list = []
        proxy = get_proxies()
        page = 0
        while True:
            page+=1
            url = 'https://sinyiwebapi.sinyi.com.tw/searchObject.php'
            header = {'User-Agent': self.user_agent.random,
                    'code':'0',
                    'sat':'730282',
                    'sid':'20230530113006065',
                    'Content-Type': 'application/json;charset=UTF-8',
                    'X-Powered-By': 'PHP/5.6.40'}
            time.sleep(random.uniform(2,6))
            ip = proxy['https']
            data = '{"machineNo":"","ipAddress":'+f'"{ip}"'+',"osType":5,"model":"web","deviceVersion":"Linux","appVersion":"111.0.0.0","deviceType":3,"apType":3,"browser":1,"memberId":"","domain":"www.sinyi.com.tw","utmSource":"","utmMedium":"","utmCampaign":"","utmCode":"","requestor":1,"utmContent":"","utmTerm":"","sinyiGroup":1,"filter":{"exludeSameTrade":false,"objectStatus":0,"retType":1,"retRange":["11"]},"page":'+str(page)+',"pageCnt":20,"sort":"2","isReturnTotal":true}'
            s = requests.Session()
            res = s.post(url, data=data, headers=header, proxies=proxy)
            print(f'開始擷取{self.source} {city} P{page} 物件')
            page_list = res.json()['content']['object']
            if not page_list:
                break
            for house in page_list:
                house['source_id'] = house.pop('houseNo')
                house['subject'] = house.pop('name')
                house['link'] = house.pop('shareURL')
                house['re_price'] = house.pop('totalPrice')
                house['city'] = city
                del house['image']
                del house['houselandtypeShow']
                del house['tags']
                # 有些沒停車的就是null
                try:
                    house['parking'] = house['parking'][0]
                except:
                    pass
                try:
                    house['houselandtype'] = house['houselandtype'][0]
                except:
                    pass
                house_list.append(house)

        if house_list:
            to_sqlite(self, house_list)
        else: # 是空的也要建立空資料表
            con = sqlite3.connect(os.path.abspath('compare.db'))
            table_name = f'{self.name}_{time.strftime("%m%d")}'
            sql = f'''
            CREATE TABLE IF NOT EXISTS {table_name} (
                "latitude"	REAL,
                "longitude"	REAL,
                "largeImage"	TEXT,
                "imageTag"	TEXT,
                "commId"	TEXT,
                "commName"	TEXT,
                "discount"	INTEGER,
                "address"	TEXT,
                "age"	TEXT,
                "houselandtype"	TEXT,
                "priceFirst"	INTEGER,
                "areaBuilding"	REAL,
                "areaLand"	REAL,
                "isHasBalcony"	INTEGER,
                "pingUsed"	REAL,
                "layout"	TEXT,
                "floor"	TEXT,
                "totalfloor"	TEXT,
                "isParking"	INTEGER,
                "parking"	TEXT,
                "threeMonthsClicks"	INTEGER,
                "isOff"	INTEGER,
                "isSimilar"	INTEGER,
                "status"	INTEGER,
                "kind"	INTEGER,
                "uniPrice"	TEXT,
                "objectType"	INTEGER,
                "3DVR"	INTEGER,
                "zipCode"	TEXT,
                "groupCompany"	TEXT,
                "addLayout"	TEXT,
                "totalLayout"	TEXT,
                "isHasVideo"	INTEGER,
                "isHasView"	INTEGER,
                "source_id"	TEXT,
                "subject"	TEXT,
                "link"	TEXT,
                "re_price"	INTEGER,
                "city"	TEXT
                );
            '''
            con.execute(sql)
            con.close()

    def get_data(self, city_code):
        data_list = []
        list_obj = is_new(self, city_code)
        for obj in list_obj:
            try:
                source_id = obj['source_id']
                link = obj['link']
                subject = obj['subject']
                total = obj['re_price']
                city = obj['city']
                print(subject, link)
                lat = round(obj['latitude'],7)
                lng = round(obj['longitude'],7)
                img_url = obj['largeImage']
                address = obj['address'].replace('台','臺')
                land_ping = obj['areaLand']
                total_ping = obj['areaBuilding']
                pattern = obj['layout']
                pattern1 = obj['addLayout']
                floor_web = obj['floor']
                total_floor = obj['totalfloor']
                house_age = obj['age']
                community = obj['commName']
                # 其他資訊須到物件頁面
                if len(data_list) % 10 == 0:
                    proxy = get_proxies()
                ip = proxy['https']
                data = '{"machineNo":"","ipAddress":'+f'"{ip}"'+',"osType":5,"model":"web","deviceVersion":"Linux","appVersion":"111.0.0.0","deviceType":3,"apType":3,"browser":1,"memberId":"","domain":"www.sinyi.com.tw","utmSource":"","utmMedium":"","utmCampaign":"","utmCode":"","requestor":1,"utmContent":"","utmTerm":"","sinyiGroup":1,"houseNo":'+ f'"{source_id}"' +',"agentId":"","memberPhone":"","showOff":0}'
                header = { 'User-Agent' : self.user_agent.random ,
                        'sat': '730282',
                        'sid': '20230530113006065',
                        'code': '0'
                        }
                s = requests.Session()
                # 紀錄物件基本資料的api
                api_url = 'https://sinyiwebapi.sinyi.com.tw/getObjectContent.php'
                time.sleep(random.uniform(7,18))
                res = s.post(api_url, headers=header, timeout=(5,30), data=data, proxies=proxy)
                res.raise_for_status()
                obj_content = res.json()['content']
                area = obj_content['zipName']
                contact_man = obj_content['agent']['agentName']
                store = obj_content['agent']['agentStore']
                phone = obj_content['agent']['agentTel']
                manage_type = obj_content['hasmanager']
                manage_fee = obj_content['monthlyFee']
                parking_type = obj_content['parking']
                is_dark = obj_content['sfdarkroom']
                is_side = obj_content['sfside']
                blockto = obj_content['buildingFront']
                # toiletplus = obj_content['bathroomplus']
                # hallplus = obj_content['hallplus']
                # roomplus = obj_content['roomplus']
                # openroomplus = obj_content['openroomplus']
                houselandtype = obj_content['houselandtype']
                pings = obj_content['areaInfo']
                agent_id = obj_content['agent']['agentId']

                # 第二隻api紀錄謄本資訊跟特色等其他詳細資料
                api_url2 = 'https://sinyiwebapi.sinyi.com.tw/getObjectDetail.php'
                time.sleep(random.uniform(6,12))
                res2 = s.post(api_url2, headers=header, timeout=(5,30), data=data, proxies=proxy)
                obj_detail = res2.json()['content']
                situation = obj_detail['detail']['purpose']
                house_num = obj_detail['detail']['family']
                feature = ','.join(obj_detail['description'])

                # 將抓下來的資料做清理
                if not situation:
                    situation = ''
                if not house_num:
                    house_num = ''
                if not blockto:
                    blockto = ''
                if not community:
                    community = ''
                if not parking_type:
                    parking_type = ''
                # 屋齡
                if house_age == '--' or house_age == '預售':
                    house_age_v = 0
                else:
                    house_age_v = float(house_age.rstrip('年'))
                # 管理費
                if not manage_type:
                    manage_type = ''
                if not manage_fee:
                    manage_fee = ''
                else:
                    manage_fee = manage_fee + '元/月'
                # 房仲
                contact_type = '仲介'
                if '(' in store:
                    branch, company = store.split('(')
                    company = company.rstrip(')')
                else:
                    branch = store
                    company = ''
                if agent_id:
                    brand = '信義房屋'
                else:# 非信義房屋的物件，brand要到物件網頁找
                    obj_header = {'User-Agent': self.user_agent.random}
                    time.sleep(random.uniform(3,8))
                    obj_res = s.get(link, headers=obj_header, proxies=proxy)
                    obj_soup = BeautifulSoup(obj_res.text, 'lxml')
                    is_brand = obj_soup.find('div',{'class':'buy-content-store-title'})
                    brand = is_brand.text.split('-')[0] if is_brand else ''
                # 邊間/暗房
                if is_dark == 'true':
                    dark = '是'
                else:
                    dark = '否'
                if is_side == 'true':
                    edge = '是'
                else:
                    edge = '否'
                # 格局
                if not pattern1:
                    pattern1 = ''
                if not pattern:
                    pattern = ''
                # pattern1 = ''
                # if roomplus >0:
                #     pattern1 += f'{roomplus}房'
                # if hallplus >0:
                #     pattern1 += f'{hallplus}廳'
                # if toiletplus >0:
                #     pattern1 += f'{toiletplus}衛'
                # if openroomplus >0:
                #     pattern1 += f'{openroomplus}室'
                # 房屋類型
                if not houselandtype:
                    houselandtype = ''
                else:
                    houselandtype = houselandtype[0]
                if houselandtype == 'A':
                    house_type = '公寓'
                elif houselandtype == 'C':
                    house_type = '套房'
                elif houselandtype == 'L':
                    house_type = '大樓'
                elif houselandtype == 'M':
                    house_type = '華廈'
                elif houselandtype == 'D':
                    house_type = '透天'
                elif houselandtype == 'E':
                    house_type = '店面'
                elif houselandtype == 'I':
                    house_type = '土地'
                elif houselandtype == 'F':
                    house_type = '辦公'
                elif houselandtype == 'G':
                    house_type = '廠房'
                elif houselandtype == 'H':
                    house_type = '倉庫'
                elif houselandtype == 'J':
                    house_type = '單售車位'
                elif houselandtype == 'K':
                    house_type = '其他'
                else:
                    house_type = ''
                # 路名
                is_road = re.findall(f'{city}{area}(.+)', address)
                road = is_road[0] if is_road else ''
                # 單價
                if total_ping!= 0:
                    price_ave = round(total/total_ping,2)
                else:
                    if land_ping!=0:
                        price_ave = round(total/land_ping,2)
                    else:
                        price_ave = 0
                # 坪數
                building_ping, att_ping, public_ping = 0,0,0
                for ping in pings:
                    if ping['title'] == '主建物':
                        building_ping += ping['area']
                    elif ping['title'] == '共有部份':
                        public_ping += ping['area']
                    else:
                        if ping['title'] != '主+陽':
                            att_ping += ping['area']
                # 樓層
                if 'B' in floor_web:
                    floor = -1
                else:
                    match1 = re.findall('(^-?\d+)-?\d?', floor_web) # -4 or -1-3 or 1-3
                    if match1:
                        floor = int(match1[0])
                    else:
                        floor = 0

            except requests.exceptions.ProxyError as err:
                self.logger.error(f"Proxy Error processing link {link}: {str(err)}")
                time.sleep(random.uniform(15,30))
                proxy = get_proxies()

            except requests.exceptions.HTTPError as err:
                self.logger.error(f"HTTPError processing link {link}: {str(err)}")
                time.sleep(random.uniform(8,20))
                proxy = get_proxies()
                continue

            except Exception as err:
                self.logger.error(f"Error processing link {link}: {str(err)}")
                continue

            else:
                # 歷史資料
                his_data = {
                            "source" : self.source,
                            "source_id" : source_id,
                            "total" : str(total),
                            "subject" : subject,
                            "insert_time" : time.strftime("%Y-%m-%d"),
                            "link" : link
                            }
                history = f'[{his_data}]'.replace("'", '"')

                data = dict()
                data["source"]=self.source
                data["source_id"]=source_id
                data["subject"]=subject
                data["city"]=city
                data["area"]=area
                data["road"]=road
                data["address"]=address
                data["situation"]=situation
                data["total"]=total
                data["price_ave"]=price_ave
                data["feature"]=feature
                data["pattern"]=pattern
                data["pattern1"]=pattern1
                data["total_ping"]=total_ping
                data["building_ping"]=building_ping
                data["public_ping"]=public_ping
                data["att_ping"]=att_ping
                data["land_ping"]=land_ping
                data["house_age"]=house_age
                data["house_age_v"]=house_age_v
                data["floor_web"]=floor_web
                data["floor"]=floor
                data["total_floor"]=total_floor
                data["house_num"]=house_num
                data["blockto"]=blockto
                data["house_type"]=house_type
                data["manage_type"]=manage_type
                data["manage_fee"]=manage_fee
                data["edge"]=edge
                data["dark"]=dark
                data["parking_type"]=parking_type
                data["lat"]=lat
                data["lng"]=lng
                data["link"]=link
                data["img_url"]=img_url
                data["contact_type"]=contact_type
                data["contact_man"]=contact_man
                data["phone"]=phone
                data["brand"]=brand
                data["branch"]=branch
                data["company"]=company
                data["price_renew"]=0
                data["insert_time"]=time.strftime("%Y-%m-%d %H:%M:%S")
                data["update_time"]=time.strftime("%Y-%m-%d %H:%M:%S")
                data["community"]=community
                data["mrt"]=''
                data["group_man"]=0
                data["group_key"]=''
                data["group_record"]=''
                data["history"]=history
                data["address_cal"]=''
                data["is_delete"]=0
                data["is_hidden"]=0
                # print(data)
                data_list.append(clean_data(data))

                # 如果超過100筆物件，就先寫入資料庫，並清空data_list
                if len(data_list) % 100 == 0:
                    data2sql(data_list, city_code)
                    data_list = []

        data2sql(data_list, city_code)
        # 把物件都寫進資料庫之後，再寫入group_key跟address_cal
        find_group(self, city_code)
        find_possible_addrs(self, city_code)


import calendar
import datetime

from django.db.models import Count
from django.template.loader import render_to_string
from django.utils.safestring import mark_safe

HISTOGRAM_CSS = """
.histogram ul {
    font-size: 0.75em;
    height: 10em;
  }

.histogram li {
    position: relative;
    float: left;
    width: 1.5em;
    margin: 0 0.1em;
    height: 8em;
    list-style-type: none;
}

.histogram li a {
    display: block;
    height: 100%;
}

.histogram li .label {
    display: block;
    position: absolute;
    bottom: -2em;
    left: 0;
    background: #fff;
    width: 100%;
    height: 2em;
    line-height: 2em;
    text-align: center;
}

.histogram li a .count {
    display: block;
    position: absolute;
    bottom: 0;
    left: 0;
    height: 0;
    width: 100%;
    background: #AAA;
    text-indent: -9999px;
    overflow: hidden;
}

.histogram li:hover {
    background: #EFEFEF;
}

.histogram li a:hover .count {
    background: #2D7BB2;
}"""

class Histogram(object):
    def __init__(self, model, attname, queryset=None, months=None, days=None):
        # `queryset` exists so it can work with the admin (bad idea?)
        self.model = model
        self.attname = attname
        self._queryset = None
        assert months or days, 'You must pass either months or days, not both.'
        self.months = months
        self.days = days

    def render(self, css=False, day_labels=True):
        context = self.get_report()
        context['day_labels'] = day_labels
        if css:
            context['css'] = HISTOGRAM_CSS
        return render_to_string("histograms/report.html", context)

    def get_query_set(self):
        return self._queryset or self.model.objects.all()

    def get_css(self):
        return mark_safe(HISTOGRAM_CSS)

    def get_report(self):
        months = {}
        if self.months:
            this_month = datetime.date.today().replace(day=1)
            last_month = this_month
            for m in xrange(self.months):
                cutoff = last_month
                months['%s.%s' % (last_month.month, last_month.year)] = [
                    last_month,
                    ([0] * calendar.monthrange(last_month.year, last_month.month)[1]),
                    0
                ]
                last_month = (last_month - datetime.timedelta(days=1)).replace(day=1)
            grouper = lambda x: '%s.%s' % (x.month, x.year)
            day_grouper = lambda x: x.day-1
        elif self.days:
            cutoff = datetime.datetime.now() - datetime.timedelta(days=self.days)
            grouper = lambda x: None
            day_grouper = lambda x: (datetime.datetime.now() - x).days
            months[None] = ['Last %s Days' % (self.days), ([0] * self.days), 0]

        qs = self.get_query_set().values(self.attname).annotate(
            num=Count("pk")
        ).filter(**{"%s__gt" % str(self.attname): cutoff})

        for data in qs.iterator():
            idx = grouper(data[self.attname])
            months[idx][1][day_grouper(data[self.attname])] += data["num"]
            months[idx][2] += data["num"]

        total = sum(o for m in months.itervalues() for o in m[1])
        max_num = max(o for m in months.itervalues() for o in m[1])
        if not (total and max_num):
            ratio = 0
        else:
            ratio = total / max_num * 100

        return {
            "results": months.values(),
            "total": total,
            "ratio": ratio
        }