def S(x):
  return x
  # return set(x)

from Common import *
from itertools import chain
import time
import sys

START = time.time()  # hack to avoid passing initial time around ....

# a unit test :)
# w = []
# w.append(Id("singSong", True))
# w.append(Id("readFile", True))
#   p = predict_dangerous_counts(w, 0.0)
# or
#   term_map = make_predictions(terms, InputChar, DIGITS)
#   p = predict_dangerous_model(w, term_map, 0.0)
# print(p)
# exit(0)

# global (ack) formats for writing data ...
# using fmt_csv ensures that all csv output has the same format :)
#
fmt = " %3d %3d %3d %3d K = %4d, max f2 = %5.3f at threshold = %5.2f (f1 = %5.3f)"
fmt_csv = "## %4.2f,%3d,%3d,%3d,%3d,%4d,  %5.3f,%5.2f,%5.3f,%s, %d,%d, %s"

def _elapse():
  return(int(time.time() - START))

def elapse(start):
  return(int(time.time() - start))

last_time = -1
def tracer(msg, time_stamp):
  global last_time
  if time_stamp - last_time > 60:  # its been a minute :)
    last_time = time_stamp
    print(msg, time_stamp, file=sys.stderr)

def trace(method, threshold, stats):
    print("%s, %5.2f, %4d, %4d, %4d, %4d, %5.3f, %5.3f"
     % (method, threshold, stats[0], stats[1], stats[2], stats[3], stats[4], stats[5]))

def compute_score(ids, dangerous, threshold):
  # return compute_score_set(ids, dangerous, threshold)
  return compute_score_ranks(ids, dangerous, threshold)

## to test if an id has one of more dangerous parts use threshold of 0.0
## (i.e., id.score > 0.0)
def compute_score_set(ids, dangerous, threshold):
    ds = set(dangerous)
    for id in ids:
        unique_parts = set(id.split)
        dangerous_parts = unique_parts & ds
        id.score = len(dangerous_parts) / len(unique_parts)
    x = compute_metrics(ids, threshold)  #     return [tp, fp, tn, fn, f2, f1]
    # x[4] = x[5]  # let's try f1
    return x

## this version considers the rank of the terms on the dangerous words list
def compute_score_ranks(ids, dangerous, threshold):
    l = len(dangerous)
    # print("\n\ntop of compute score for dangerous = ", dangerous, "threshold = ", threshold)
    for id in ids:
        unique_parts = set(id.split)
        # print("\nid = ", id.id, " unique parts = ", unique_parts)
        score = 0
        for part in unique_parts:
            # r = rank(part, dangerous)
            try:
              r = dangerous.index(part)
              # print("\nxx score for", part, " = ", l-r)
            except ValueError:
              # print("OOPS setting r = l")
              r = l
            score += (l-r) / l   # rank 0 -> += 1.0
            # print("  score for  ", part, " = ", (l-r)/l, " ... current sum ", score)

        # print("post loop score = ", score, "len = ", len(unique_parts))
        id.score = score / len(unique_parts)
        # print("final score for ", id.id, " is ", id.score)
    x = compute_metrics(ids, threshold)  #     return [tp, fp, tn, fn, f2, f1]
    # x[4] = x[5]  # let's try f1
    return x

def all_dangerous(test):
  stats =  compute_metrics(test, -1) # predict everything is dangerous
  # stats[4] = stats[5]    # let's try f1
  return stats

# note that make_predictions is called once generating term map that is
# subseted here.  for folds=5 doing so was 4x faster!
def predict_dangerous_model(identifiers, term_map, min_score):
  terms = gather_terms(identifiers)
  score_map = {t: term_map[t] for t in terms}

  ## order dangerous terms based on score
  listofTuples = sorted(score_map.items(), key=lambda x: -x[1])
  dangerous_words = [elem[0] for elem in listofTuples if elem[1] >= min_score]
  # for elem in listofTuples:
    # print("%5.3f %5.3f %s" % (min_score, elem[1],elem[0]))
  # for i in range(0, len(dangerous_words)):
    # print("%5.3f %s %s" % (listofTuples[i][1],listofTuples[i][0], dangerous_words[i]))

  ## dump top and bottom 200 words
  # print("@@ M marker", len(terms), len(listofTuples),  file=sys.stderr)
  # print("for (min score %d ) top 200 scores" % (min_score))
  # for i in range(0, 200):
  #   print("%5.3f %s %s" % (listofTuples[i][1],listofTuples[i][0], dangerous_words[i]))
  # print("for (min score %d ) bottom 200 scores" % (min_score))
  # for i in range(len(dangerous_words)-200, len(dangerous_words)):
  #   print("%5.3f %s %s" % (listofTuples[i][1],listofTuples[i][0], dangerous_words[i]))

  print("@@ ids %d, unique terms %d, map-size %d, dw %d %d" % (len(identifiers), len(set(terms)), len(score_map), len(dangerous_words), len(set(dangerous_words))))
  return (dangerous_words)

def predict_dangerous_counts(identifiers, min_score, plus, minus):
  terms = {}
  for id in identifiers:
    for t in id.split:
      if not t in terms:
        terms[t] = 0

      if id.vulnerable:
        terms[t] += plus
      else:
        terms[t] -= minus

  # print("number of terms ", len(terms))
  # listofTuples = sorted(terms.items(), key=lambda x: -x[1])
  # for i in range(0, len(listofTuples)):
    # print("%5.3f %s" % (listofTuples[i][1],listofTuples[i][0]))
  # exit(0)

  terms = {k:v for k,v in terms.items() if min_score < -900 or v >= min_score}

  ## order dangerous terms based on score
  listofTuples = sorted(terms.items(), key=lambda x: -x[1])
  dangerous_words = [elem[0] for elem in listofTuples ]
  # for elem in listofTuples:
    # print("%5.3f %s" % (elem[1],elem[0]))
  # for i in range(0, len(dangerous_words)):
    # print("%5.3f %s %s" % (listofTuples[i][1],listofTuples[i][0], dangerous_words[i]))

  ## dump top and bottom 200 words
  # print("@@ M marker", len(terms), len(listofTuples),  file=sys.stderr)
  # print("for (plus %d minus %d) top 20 scores" % (plus, minus))
  # for i in range(0, 20):
  #   print("%5.3f %s %s" % (listofTuples[i][1],listofTuples[i][0], dangerous_words[i]))
  # print("for (plus %d minus %d) bottom 20 scores" % (plus, minus))
  # for i in range(len(dangerous_words)-20, len(dangerous_words)):
  #   print("%5.3f %s %s" % (listofTuples[i][1],listofTuples[i][0], dangerous_words[i]))

  # print("@@ ids %d, unique terms %d, map-size %d, dw %d" % (len(identifiers), len(set(terms)), len(dangerous_words), len(set(dangerous_words))))
  return (dangerous_words)

# i/j for j = 1 to 6
# thresholds = [0.17, 0.20, 0.25, 0.33, 0.40, 0.50, 0.60, 0.67, 0.75, 0.80, 0.83, 1.00]
# computation uses "> threshold", so subtract 0.01 from each (and add -0.01)
thresholds = [ 0.01, 0.16, 0.19, 0.24, 0.32, 0.39, 0.49, 0.59, 0.66, 0.74, 0.79, 0.82, 0.99]
# the following enables the algorithm to choose 'all dangerous' as an option.
# current data does not include this, so we can see how bad the search truly is :)
# array.insert(-0.01, thresholds)

def best_thresh(identifiers, d, K, ret):
  # OLD: full range is 0.00 to < 0.99, but for the model values outside [0.15, 0.33]
  # OLD: rarely if ever win the day so its faster to use [0.10, 0.40]
  for threshold in thresholds:
    stats = compute_score(identifiers, d, threshold)
    # FYI f1 is in stats[5]
    if (ret[0] < stats[4]):
      # print("f2 score upgrade from ", ret[0], " to ", stats[4])
      ret = [stats[4], threshold, K, stats[0], stats[1], stats[2], stats[3]]
    # else:
      # print("f2 score stay ", ret[0], " > ", stats[4], " for K = ", K, " and threshold = ", threshold)

  return ret

# this needs work (and the idea kinda replicates the slow search)
# current version always set K somewhere near the end :)
def new_find_best_K_n_threshold(dangerous, identifiers, fold):
  start = 1
  stop = len(dangerous) + 1
  step = 2
  stepupat = 10

  ret = [-1, -1, -1]

  # print("@@ start", start, "stop", stop, "step", step,  file=sys.stderr)
  while start < stop:
    # print("start ", start, "step", step, "stepupat ", stepupat, "stop", stop, file=sys.stderr)
    d = S(dangerous[0:start])
    ret = best_thresh(identifiers, d, start, ret)

    start = start + step
    if start > stepupat:
      step = step * 10
      stepupat = stepupat * 10

    if start + step > stop:
      step = int(step / 10)
    if step == 0:
      step = 1

  step = int(step/10)
  steupauat = int(stepupat/10)
  print("@@ find final K ", start, " final step ", step, "final step-up-at ", stepupat, file=sys.stderr)

  return ret

# based on f2 score and training data
# for 2200 identifiers profiler find this taking
#  46.647    4.665 CrossValidation.py:99(find_best_K_n_threshold)
# compared to
#  3.184    0.318 CrossValidation.py:129(find_best_K_n_threshold)
# for the binary search version
#
def slow_find_best_K_n_threshold(dangerous, identifiers, fold):
  start = 1
  max = len(dangerous) + 1
  stop = max
  step = int((max-1)/20)  # goal 20 steps
  if step < 1:
    step = 1

  ret = [-1, -1, -1]

  # step = 1 # FIX ME

  ## note that some local maxes in training produce better test output
  ## making working hard to get best training answer not worth it :(
  print("@@ start", start, "stop", stop, "step", step,  file=sys.stderr)
  # last iteration makes a notable improvement to f2 (avoid while step > 9)
  while step >= 1:
    print("  fb  at start", start, "stop", stop, "step", step, _elapse(), file=sys.stderr)
    # chain ensures that stop is included
    for K in chain(range(start, stop, step), range(stop-1, stop)):
      d = S(dangerous[0:K])
      ret = best_thresh(identifiers, d, K, ret)
      # print("slow find  K %d  winning f2 %5.4f   at %5.3f, %d    start %d step %d stop %d" % (K, ret[0], ret[1], ret[2], start, step, stop), file=sys.stderr)

    start = ret[2] - step
    if start < 1:
      start = 1
    stop = ret[2] + step
    if stop > max:
      stop = max
    step = int(step / 10)

  return ret

# based on f2 score and training data
# binary search approach ... alas landscape includes local maxes, so this is approximate
def old_find_best_K_n_threshold(dangerous, identifiers, fold):
  start = 1
  stop = len(dangerous) + 1
  mid = int((stop+start)/2)
  ret = [-1, -1, -1]
  l = best_thresh(identifiers, S(dangerous[0:start]), start, ret)
  h = best_thresh(identifiers, S(dangerous[0:stop]), stop, ret)

  while start < mid and mid < stop:
    if stop - start > 1000:
      tracer("  fb at start %d stop %d mid %d " % (start, stop, mid), _elapse())

    if l[0] > h[0]:
      stop = mid
      h = best_thresh(identifiers, S(dangerous[0:stop]), stop, ret)
    else:
      start = mid
      l = best_thresh(identifiers, S(dangerous[0:start]), start, ret)
    mid = int((stop+start)/2)

  tracer("  fb post loop at start %d stop %d mid %d " % (start, stop, mid), _elapse())
  return l if l[0] > h[0] else h

# the expensive n^2-ish approach that produces 1-2% improvement but takes 3x the execution time
def model_search(dangerous, identifiers, fold, plus, minus):
  best_f1 = -1
  best_f2 = -1
  best_f2_msg = "## oops len(dangerous) is %d (if this is not zero something is very wrong :( " % len(dangerous)

  if len(dangerous) > 1000:
    step = int(len(dangerous)/20)
    step = 100    # bulk of the data generation used 100 ... FIX ME reconsider??
  else:
    step = 1

  start = 1
  stop = len(dangerous) + 1
  for K in range(start, stop, step):
    if (K-1) % 2000 == 0:
      print("  ms time at start of K= ", K,  _elapse(), file=sys.stderr)
    d = S(dangerous[0:K])

    max = [-1, -1, -1, -1, -1]
    maxat = -1
    threshold = -0.02 # figures used is -0.01 ... was 0.00
    while threshold < 0.99:
      threshold += 0.01
      stats = compute_score(identifiers, d, threshold)
      maxat = maxat if max[4] >= stats[4] else threshold
      max = max if max[4] >= stats[4] else stats
      print(fmt_csv
           % (threshold, stats[0], stats[1], stats[2], stats[3], K, stats[4], maxat, stats[5], fold, plus, minus, "graph"))

    # print(" %3d %3d %3d %3d K = %4d, max f2 = %5.3f at threshold = %5.2f (f1 = %5.3f)"
    #       % (max[0], max[1], max[2], max[3], K, max[4], maxat, max[5]))

    if max[4] > best_f2:
      best_f2 = max[4]
      best_f2_msg = fmt_csv % (maxat, max[0], max[1], max[2], max[3], K, max[4], maxat, max[5], fold, plus, minus,"")
      # print("PING " + best_f2_msg)
    if max[5] > best_f1:
      best_f1 = max[5]
      best_f1_msg = fmt_csv % (maxat, max[0], max[1], max[2], max[3], K, max[4], maxat, max[5], fold, plus, minus,"")
    # break
    # exit(0)

  print(best_f2_msg, ", @@ model search best-f2")
  # print("@@", best_f1_msg, ", @best-f1")

def dump(category, id, v, b, threshold):
  s = ""
  for p in id.split:
    if p in v:
      s = s + "(v " + p + ")"
    elif p in b:
      s = s + "(b " + p + ")"
    else:
      s = s + "(. " + p + ")"
  if category != "tn" or id.score != 0.00:
    print(category, "%5.2f" % id.score, threshold, "  ", id.id, s)

# look at overlap between training and test vulnerable and benign identifiers
def termOverlap(dw, training, test, threshold):
  vtrain = list()
  vtest = list()
  btrain = list()
  btest = list()

  # x = set([id.id for id in test])
  # print("test = ", x)
  # x = set([id.id for id in training])
  # print("training = ", x)

  for id in training:
    for p in id.split:
      if p in dw:
        vtrain.append(p)
      else:
        btrain.append(p)

# way 2
#   for id in training:
#     if id.vulnerable:
#       for p in id.split:
#         vtrain.append(p)
#     else:
#       for p in id.split:
#         btrain.append(p)
#
#   for id in test:
#     if id.vulnerable:
#       for p in id.split:
#         vtest.append(p)
#     else:
#       for p in id.split:
#         btest.append(p)

  # print("vtrain = ", vtrain)
  vtrain = set([x for x in vtrain if len(x) > 1])
  btrain = set([x for x in btrain if len(x) > 1])
  sv = len(vtrain)
  sb = len(btrain)
  # v = len(vtrain.intersection(vtest))   # use .add() with sets....
  # b = len(btrain.intersection(btest))

  v = len([x for x in vtrain if x in vtest])
  b = len([x for x in btrain if x in btest])
  print("xx vul train %3d, test %3d,  ben train %3d, test %3d, " % (len(vtrain), len(vtest), len(btrain), len(btest)), end= " ")
  print("overlaps: vul %3d %6.3f%%,  ben %3d %6.3f%%\n"  % (v, (100*v/sv), b, (100*b/sb)))

  for id in test:
    if id.score > threshold and id.vulnerable:
      dump("tp", id, vtrain, btrain, threshold)
    if id.score > threshold and not id.vulnerable:
      dump("fp", id, vtrain, btrain, threshold);
    if id.score <= threshold and id.vulnerable:
      dump("fn", id, vtrain, btrain, threshold);
    if id.score <= threshold and not id.vulnerable:
      dump("tn", id, vtrain, btrain, threshold);

def score(training, test, score_min, term_map, name, f2_all_ids, plus, minus):
  # print("# for sibren ... run the NN 'word finder' portion of the algorithm on the whole of test+train")
  # training = training + test
  print("# training size %d, test size %d\n" % (len(training), len(test)))
  print("\ntime at start of fold ", name,  elapse(START), file=sys.stderr)

  # termOverlap(FIXME, training, test, 0.00)
  # return

  stats = all_dangerous(training)
  print(fmt_csv % ( -1, stats[0], stats[1], stats[2], stats[3], -1, stats[4], -1, stats[5], name, plus, minus, "all dangerous train"))
  f2_ad_train = stats[4]

  stats = all_dangerous(test)
  print(fmt_csv % ( -1, stats[0], stats[1], stats[2], stats[3], -1, stats[4], -1, stats[5], name, plus, minus, "all dangerous test"))
  f2_ad_test = stats[4]

  if term_map == None:
    # prelim pms = [(3,4), (1,1), (3,2), (7,4), (2,1), (9,4), (5,2), (11,4), (3,1), (7,2), (4,1), (5,1)]
    # VDISC would have picked (10,1)
    if plus == 0 and minus == 0:
      # Most paper data omits (10,1).  Analysis suggested trying 10,1 with
      # VDISC, which worked well.  Older data was not regenerated.
      pms = [(3,4), (1,1), (3,2), (7,4), (2,1), (9,4), (5,2), (11,4), (3,1), (7,2), (4,1), (5,1), (10,1)]
    else:
      pms = [(plus, minus)]   # for weight study use
  else:
    pms = [(0,0)]  # effectively unused

  f2_best = [-1,-1,-1]
  for p, m in pms:
    print("@@ p,m main loop (plus %d minus %d)" % (p, m), file=sys.stderr)
    if term_map == None:
      dangerous = predict_dangerous_counts(training, score_min, p, m)
    else:
      dangerous = predict_dangerous_model(training, term_map, score_min)
    # print("time pre find_best ", name,  elapse(START), file=sys.stderr)
    print("@@ there are %d dangerous words" % (len(dangerous)), file=sys.stderr)
    v =  slow_find_best_K_n_threshold(dangerous, training, name)
    # v =  old_find_best_K_n_threshold(dangerous, training, name)
    # print("@@ back in score bi search return selected K",v[2])
    # print(fmt_csv % (v[1], v[3], v[4], v[5], v[6], v[2], v[0], -1, -1, name, p, m, "@@ find best training (plus %d minus %d)" % (p, m)))
    f2_best_training = v[0]
    if f2_best_training >= f2_best[0]:
      f2_best = [f2_best_training, p, m]
      v_best = v

  p = "fish"
  m = "fish"
  v = v_best
  f2_best_training = f2_best[0]
  plus = f2_best[1]
  minus = f2_best[2]
  # print("@@ training produced best f2 of %5.3f at plus=%d, minus=%d\n" % (f2_best_training, plus, minus))
  print(fmt_csv % (v[1], v[3], v[4], v[5], v[6], v[2], v[0], -1, -1, name, plus, minus, "@@ find_best(training)"))

  if term_map == None:   # if recompute takes too long stash this
    dangerous = predict_dangerous_counts(training, score_min, plus, minus)
  print("@@ using %d of the %d dangerous words" % (v[2], len(dangerous)))

  if len(dangerous) == v[2]:
    print("@@@ starved! taking all words as dangerous")

  t1 = time.time()
  # FIX ME V2 might suppress words of length 1
  # d = S([x for x in dangerous[0:v[2]] if len(x) > 1])
  d = S(dangerous[0:v[2]])
  # print(d)

  threshold = v[1]
  stats = compute_score(test, d, threshold)
  # termOverlap(d, training, test, threshold)
  print("@@ predicting time ", ((time.time() - t1)*1000), "ms for", len(test), "all_ids")
  print(fmt_csv
     % (threshold, stats[0], stats[1], stats[2], stats[3], v[2], stats[4], v[1], stats[5], name, plus, minus,
       "@@ eval test s"))
  f2_predicted = stats[4]

  # for folds=5, test is 1/5 the size of training, so this cost is ~80% less
  v =  slow_find_best_K_n_threshold(dangerous, test, name)
  # v =  old_find_best_K_n_threshold(dangerous, test, name)
  print(fmt_csv % (v[1], v[3], v[4], v[5], v[6], v[2], v[0], -1, -1, name, plus, minus, "@@ find_best(test)"))
  f2_best_test = v[0]

  # this call is real slow :)  (see keepers/45-test)
  # model_search(dangerous, test, name, plus, minus)     # also generates data for graphs
  print("@@ *** warning final model search commented out ***")

  print("")

  print("### AD, AD train, AD test, best train, predicted, best test, data set, dwc")
  print("### %6.3f, %6.3f, %6.3f, %6.3f,  %6.3f,  %6.3f, %s, %d, %d, %d" % (f2_all_ids, f2_ad_train, f2_ad_test, f2_best_training, f2_predicted, f2_best_test, name, len(dangerous), plus, minus))


from flask import Flask, Blueprint
from flask_jwt_extended import JWTManager
from mail import mail
from db import db
from slack import slack_client
from discord import discord_client

import json

class ConfigurationError(Exception):
    """Exception raised for errors in app config
    """

    def __init__(self, key):
        super().__init__("\"{}\" not found in application config".format(key))

def get_opt_config(app, config, key):
    if (key in config):
        app.config[key] = config[key]

def get_req_config(app, config, key):
    if (key not in config): raise ConfigurationError(key)
    app.config[key] = config[key]

def create_app(config_file='config/config.json'):
    app = Flask(__name__)

    config = json.load(open(config_file))

    get_req_config(app, config, 'DEBUG')
    app.debug = app.config['DEBUG']

    get_req_config(app, config, 'SECRET_KEY')
    get_req_config(app, config, 'MONGO_URI')
    get_req_config(app, config, 'MONGO_DB_NAME')
    get_req_config(app, config, 'TESTING')
    get_req_config(app, config, 'BASE_URL')
    get_opt_config(app, config, 'SLACK_WEBHOOK')
    get_opt_config(app, config, 'DISCORD_WEBHOOK')

    get_opt_config(app, config, 'MAIL_SERVER')
    get_opt_config(app, config, 'MAIL_PORT')
    get_opt_config(app, config, 'MAIL_USERNAME')
    get_opt_config(app, config, 'MAIL_PASSWORD')

    app.config['SLACK_SUPPRESS_SEND'] = False
    app.config['DISCORD_SUPPRESS_SEND'] = False

    # Supress mail sending if not specified, i.e. in dev
    if ('MAIL_SERVER' not in config):
        app.config['MAIL_SUPPRESS_SEND'] = True

    # Supress slack sending if not specified, i.e. in dev
    if ('SLACK_WEBHOOK' not in config):
        app.config['SLACK_SUPPRESS_SEND'] = True

    # Supress discord sending if not specified, i.e. in dev
    if ('DISCORD_WEBHOOK' not in config):
        app.config['DISCORD_SUPPRESS_SEND'] = True

    # Configurations that are always the same
    app.config['JWT_TOKEN_LOCATION'] =  ['cookies', 'headers']
    app.config['JWT_REFRESH_COOKIE_PATH'] = '/api/auth/session'
    app.config['JWT_COOKIE_CSRF_PROTECT '] = False
    app.config['JWT_CSRF_IN_COOKIES'] = False
    app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024

    # Pass configurations to extensions
    jwt = JWTManager(app)
    db.init_app(app)
    mail.init_app(app)
    slack_client.init_app(app)
    discord_client.init_app(app)

    # Add endpoints from these files
    # Note order is important here
    from auth import auth_api
    from accounts import accounts_api
    from admin import admin_api
    from resumes import resume_api
    from vaccination import vaccination_api
    from registrations import registrations_api
    from assign import assign_api
    from announcements import announcements_api
    from events import events_api
    from slack import slack_api
    from discord import discord_api
    from teammatch import teammatch_api

    app.register_blueprint(auth_api, url_prefix='/api/auth')
    app.register_blueprint(admin_api, url_prefix='/api/admin')
    app.register_blueprint(accounts_api, url_prefix='/api/accounts')
    app.register_blueprint(resume_api, url_prefix='/api/resumes')
    app.register_blueprint(vaccination_api, url_prefix='/api/vaccination')
    app.register_blueprint(registrations_api, url_prefix='/api/registrations')
    app.register_blueprint(assign_api, url_prefix='/api/judgetool')
    app.register_blueprint(announcements_api, url_prefix='/api/announcements')
    app.register_blueprint(events_api, url_prefix='/api/events')
    app.register_blueprint(slack_api, url_prefix='/api/slack')
    app.register_blueprint(discord_api, url_prefix='/api/discord')
    app.register_blueprint(teammatch_api, url_prefix='/api/teammatch')

    return app


"""This module contains the main training loop"""

from random import random

import numpy as np
import torch
from scipy.spatial.transform import Rotation
from torch.utils.data import DataLoader
from tqdm import tqdm

from pointcloud_invariance_smoothing.models.invariance_wrappers import (
    PCAEnsembleWrapper, SinglePCAWrapper, StackingPCAWrapper)
from pointcloud_invariance_smoothing.models.pointnet import (
    PointNetClassifier, pointnet_loss)

def train(model, data_train, data_val,
          num_epochs, batch_size,
          learning_rate, weight_decay, regularization_weight,
          scheduler_stepsize, scheduler_gamma,
          rotate, scale, add_noise,
          scale_limits, std,
          rotate_validation, add_noise_validation,
          device, num_workers):
    """Main training loop, for meaning of parameters see seml/scripts/train.py.

    Returns:
        Four arrays of length num_epochs, tracking train loss, val loss,
        train accuracy, val accuracy
        and two state dicts containing model with best validation loss and model
        with best validation accuracy.
    """

    train_loader = DataLoader(data_train, drop_last=True, batch_size=batch_size, shuffle=True,
                              num_workers=num_workers)

    val_loader = DataLoader(data_val, drop_last=False, batch_size=batch_size, shuffle=True,
                            num_workers=num_workers)

    optimizer = torch.optim.Adam(
            model.parameters(),
            lr=learning_rate,
            weight_decay=weight_decay
        )

    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=scheduler_stepsize,
                                                gamma=scheduler_gamma)

    loss_fn = pointnet_loss

    loss_trace_train = []
    acc_trace_train = []

    loss_trace_val = []
    acc_trace_val = []

    best_loss_val = None
    best_acc_val = None

    best_model_loss = None
    best_model_acc = None

    for _ in tqdm(range(num_epochs)):
        model.train()

        per_batch_losses = []
        per_batch_accs = []

        for X, targets in train_loader:
            optimizer.zero_grad()

            X, targets = X.to(device), targets.to(device)

            if rotate:
                X = randomly_rotate(X, device)

            if scale:
                X = randomly_scale(X, *scale_limits, device)

            if add_noise:
                X = add_gaussian_noise(X, std, device)

            logits, trans_feat = model(X)

            loss = loss_fn(logits, targets.long(), trans_feat, regularization_weight)
            loss.backward()

            optimizer.step()

            pred = logits.detach().max(dim=1)[1]
            n_correct = (pred == targets.long()).sum().cpu()
            per_batch_accs.append(int(n_correct) / batch_size)

            per_batch_losses.append(float(loss.detach().cpu()))

        loss_trace_train.append(np.mean(per_batch_losses))
        acc_trace_train.append(np.mean(per_batch_accs))

        scheduler.step()

        model.eval()

        per_batch_losses = []
        per_batch_accs = []

        for X, targets in val_loader:
            X, targets = X.to(device), targets.to(device)

            if rotate_validation:
                X = randomly_rotate(X, device)
                raise ValueError()

            if add_noise_validation:
                X = add_gaussian_noise(X, std, device)
            else:
                if add_noise:
                    raise ValueError()

            logits, _ = model(X)

            loss = loss_fn(logits, targets.long())

            pred = logits.detach().max(dim=1)[1]
            n_correct = (pred == targets.long()).sum().cpu()
            per_batch_accs.append(int(n_correct) / batch_size)

            per_batch_losses.append(float(loss.detach().cpu()))

        loss_trace_val.append(np.mean(per_batch_losses))
        acc_trace_val.append(np.mean(per_batch_accs))

        print(loss_trace_val[-1], acc_trace_val[-1])

        if isinstance(model, (PCAEnsembleWrapper, SinglePCAWrapper, StackingPCAWrapper)):
            state_dict = model.base_model.state_dict()
        else:
            state_dict = model.state_dict()

        if best_loss_val is None or loss_trace_val[-1] < best_loss_val:
            best_loss_val = loss_trace_val[-1]
            best_model_loss = state_dict
        if best_acc_val is None or acc_trace_val[-1] < best_acc_val:
            best_acc_val = acc_trace_val[-1]
            best_model_acc = state_dict

    return (loss_trace_train, loss_trace_val,
            acc_trace_train, acc_trace_val,
            best_model_loss, best_model_acc)

def add_gaussian_noise(x: torch.Tensor, std: float, device: torch.device) -> torch.Tensor:

    return x + std * torch.randn(x.shape).to(device)

def randomly_scale(x: torch.Tensor, scale_min: float, scale_max: float,
                   device: torch.device) -> torch.Tensor:
    """Scale each point cloud in batch by a different factor sampled from uniform distribution"""

    scale = torch.rand(len(x)) / (scale_max - scale_min) + scale_min
    scale = scale[:, None, None].to(device)

    return x * scale

def randomly_rotate(x: torch.Tensor, device: torch.device) -> torch.Tensor:
    """Randomly rotate each point cloud in batch using uniformly sampled rotation matrix"""
    n_dim = x.shape[-1]
    batch_size = x.shape[0]

    rotation_matrices = generate_random_rotations(n_dim, batch_size).to(device)

    return torch.bmm(x, rotation_matrices.transpose(2, 1))

def generate_random_rotations(n_dim: int, n_samples: int) -> torch.Tensor:
    """Return n_samples different rotation matrices of shape n_dim x n_dim"""
    assert n_dim in [2, 3]

    if n_dim == 2:
        angles = np.random.uniform(0, 2 * np.pi, n_samples)

        rotation_matrices = np.empty((n_samples, n_dim, n_dim))
        rotation_matrices[:, 0, 0] = np.cos(angles)
        rotation_matrices[:, 0, 1] = -1 * np.sin(angles)
        rotation_matrices[:, 1, 0] = np.sin(angles)
        rotation_matrices[:, 1, 1] = np.cos(angles)

    if n_dim == 3:
        rotation_matrices = Rotation.random(n_samples).as_matrix()

    return torch.Tensor(rotation_matrices)


""" Wrapper script for awscli which connects to containers running in
    ecs seamlessly """
import click
import logging
import os
import re
import inquirer
import signal
from ecs_connect.version import __version__
from ecs_connect.config import ECSConfig
from ecs_connect.ecs import ECSHandler
from ecs_connect.ssm import SSMHandler
from configparser import RawConfigParser

@click.command()
@click.option('--profile', help="Name of the profile to use in ~/.ecs-connect. \
If none is provided, then the default profile will be used.\n")
@click.option('--awsprofile', help="Name of the aws profile to use form ecs-connect. \
If none is provided, then the default aws profile will be used.\n")
@click.option('--cluster', help="Name of the ECS cluster. \
If provided, then parameter from profile will be overridden.\n")
@click.option('--service', help="Name of the service. \
If provided, then parameter from profile will be overridden.\n")
@click.option('--task', help="Name of the task. \
If provided, then parameter from profile will be overridden.\n")
@click.option('--exec_cmd', help="Initilization command to run with ecs exec. \
If provided, then parameter from profile will be overridden.\n")
@click.option('--container', help="Name of the container. \
If provided, then parameter from profile will be overridden.\n")
@click.option('--bastion', help="Instance Id of the bastion node, \
required for task running in FARGATE. \
If provided, then parameter from profile will be overridden.\n")
@click.option('--cmd', help="Initilization command to run. \
If provided, then parameter from profile will be overridden.\n")
@click.option('-l', '--logs', is_flag=True, help="fetch logs.")
@click.option('--since', default='20m', help="From what time to begin displaying logs.\n")
@click.option('-f', '--follow', is_flag=True,
              help="Whether to continuously poll for new logs.\n")
@click.option('--format', default='short', help="The format to display the logs.\n")
@click.option('--filter', help="The filter pattern to use.\n")
@click.option('-a', '--all', is_flag=True,
              help='Displays all running containers\n')
@click.option('-V', '--version', is_flag=True,
              help='Displays version number\n')
@click.option('-v', '--verbose', is_flag=True, help='Enables verbose mode')
@click.option('-d', '--debug', is_flag=True, help='Enables debug mode')
@click.option('-i', '--interactive', is_flag=True, help='Interactive mode')
def main(profile, awsprofile, cluster, service, task, container, bastion, exec_cmd, cmd,
         logs, since, follow, format, filter, all, version, verbose, debug, interactive):
    if version:
        print(__version__)
        exit(0)

    # Set up logging
    logger = logging.getLogger('ecs-connect')
    logger.setLevel(logging.DEBUG)
    handler = logging.StreamHandler()
    handler.setLevel(logging.WARN)
    formatter = logging.Formatter('%(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    if verbose:
        handler.setLevel(logging.INFO)
    if debug:
        handler.setLevel(logging.DEBUG)
    logger.addHandler(handler)

    if not interactive or logs:
        ecs_config = ECSConfig(logger)
        if not profile:
            profile = "default"
        if not cluster:
            cluster = ecs_config.get_cluster(profile)
        if not awsprofile:
            awsprofile = ecs_config.get_awsprofile(profile)
        if not task:
            task = ecs_config.get_task(profile)
        if not exec_cmd:
            exec_cmd = ecs_config.get_exec_cmd(profile)
        if not container:
            container = ecs_config.get_container(profile, exec_cmd)
        if task is None:
            if not service:
                service = ecs_config.get_service(profile)
        else:
            service = None
        if not bastion:
            bastion = ecs_config.get_bastion(profile)
        if not cmd:
            cmd = ecs_config.get_cmd(profile, exec_cmd)
        ssh_user = ecs_config.get_ssh_user(profile)
        ssh_key = ecs_config.get_ssh_key(profile)
        ssh_port = ecs_config.get_ssh_port(profile)

        ecs = ECSHandler(awsprofile, cluster, service, task, container, bastion, logger, cmd, exec_cmd)
        if exec_cmd is not None:
            if logs:
                ecs.logs(since, follow, format, filter)
            else:
                ecs.exec()
        else:
            instance_id, bastion_enabled = ecs.get_ec2_instance_id()
            ssm = SSMHandler(awsprofile, instance_id, service, bastion_enabled, bastion, ssh_user, ssh_key, ssh_port, logger)
            ssm.run(cmd, all)
    else:
        config_path = os.path.expanduser('~') + '/.aws/config'
        _value = RawConfigParser()
        _value.read(config_path)
        aws_profiles = _value.sections()
        profiles = map( str, aws_profiles)
        profiles = list(map( lambda x: re.sub("profile ", "", x), profiles))
        questions = [
            inquirer.List('aws',
                        message="Select AWS Profile",
                        choices=profiles,
                    ),
        ]
        answers = inquirer.prompt(questions)
        ecs = ECSHandler(answers["aws"], '', '', '', '', '', logger, '', 'bash')
        ecs.interactive()

if __name__ == "__main__":
    main()


from telegram import (ReplyKeyboardMarkup, ReplyKeyboardRemove, ParseMode)
from telegram.ext import (Updater, CommandHandler, MessageHandler, Filters,
                          ConversationHandler)

from controllers.controllerUtils import listUtils

from models.Administrativo import Administrativo
from models.Registro import Registro

from database.main import Database
from sqlalchemy.sql import func

class CombustivelExport:
    def __init__(self, logger):
        self.logger = logger
        self.info_combustivel = CommandHandler('info_combustivel', self.get_info)

    def get_info(self, update, context):
        if update.message.from_user.username is None:
            update.message.reply_text(
                'Não é possível realizar a consulta sem um nome de usuário cadastrado.',
                reply_markup=ReplyKeyboardRemove()
            )
            return

        try:
            Session = Database.Session
            session = Session()

            administrativo = session.query(Administrativo).filter_by(
                telegram_user=update.message.from_user.username).first()

            if administrativo is None:
                update.message.reply_text(
                    'Usuário ' + update.message.from_user.username + ' não encontrado na base de dados. ' +
                    'Acesso não autorizado!', reply_markup=ReplyKeyboardRemove()
                )
                session.close()

        except Exception as e:
            print(e)
            update.message.reply_text('Houve um erro ao tentar se conectar com a base de dados! ' +
                                      'O erro foi reportado, tente novamente mais tarde.',
                                      reply_markup=ReplyKeyboardRemove())
            return

        if len(context.args) < 1:
            update.message.reply_text('Ola, ' + administrativo.nome + ' por favor, informe o ID ' +
            'da informação de combustível a ser consultada. \n\nEx: /info_combustivel 5')
        else:
            try:
                arg_id = int(context.args[0])
            except:
                update.message.reply_text('O argumento repassado deve ser um número inteiro. \n\n' +
                'Ex: /info_combustivel 20')
                session.close()
                return

            update.message.reply_text('Buscando registro...')
            registro = session.query(Registro).filter_by(id=arg_id).first()

            if registro is None:
                update.message.reply_text('Registro não encontrado :(')
                session.close()
                return

            update.message.reply_text('Enviando as fotos...')

            context.bot.send_photo(update.effective_chat.id,caption='Bomba antes do abastecimento', photo=open('media/' + registro.media_dir + '/bomba_antes.jpg', 'rb'))
            context.bot.send_photo(update.effective_chat.id,caption='Bomba depois do abastecimento', photo=open('media/' + registro.media_dir + '/bomba_depois.jpg', 'rb'))
            context.bot.send_photo(update.effective_chat.id,caption='Nota fiscal', photo=open('media/' + registro.media_dir + '/nota_fiscal.jpg', 'rb'))
            context.bot.send_photo(update.effective_chat.id,caption='Painel do veículo', photo=open('media/' + registro.media_dir + '/painel.jpg', 'rb'))
            context.bot.send_photo(update.effective_chat.id,caption='Placa do veículo', photo=open('media/' + registro.media_dir + '/placa.jpg', 'rb'))

        session.close()


from collections import defaultdict
from io import BytesIO

import pandas as pd
from flask import Flask, render_template, request

pd.set_option("display.max_colwidth", -1)

app = Flask(__name__)

@app.route("/")
def index():
    return render_template("index.html")

def _to_str(token):
    if type(token) != str:
        token = str(int(token))
    return token

def _hotel_summary(df):
    num_unique_guests = len(df["Token"][df["Token"].notna()].unique())
    end_time = pd.to_datetime(df.groupby(df["Token"])["Date/Time"].max())
    start_time = pd.to_datetime(df.groupby(df["Token"])["Date/Time"].min())
    duration = end_time - start_time

    return {
        "max_duration_token": _to_str(duration.idxmax()),
        "max_duration_minutes": duration.max().seconds // 60,
        "num_unique_guests": num_unique_guests,
    }

def _chat_summary(df):
    df[["Token"]] = df[["Token"]].fillna(method="ffill")

    data = {}
    for token, sub_df in df.groupby("Token"):
        filled = sub_df[["Guest messages", "Bebot messages"]].fillna("")
        filled["Guest messages"] = filled["Guest messages"].str.replace("\n", "
")
        filled["Bebot messages"] = filled["Bebot messages"].str.replace("\n", "
")
        data[_to_str(token)] = filled.to_html(
            index=False, escape=False, classes="table"
        )

    result = ""
    for t, table in data.items():
        result += f"
 {t}
"
        result += table
    return result

@app.route("/result", methods=["POST"])
def generate_result():
    file = request.files["file"]
    if not file.filename.endswith("xlsx"):
        return "Please upload an excel spreadsheet"
    content_bytes = BytesIO(file.read())

    dfs = pd.read_excel(content_bytes, engine="xlrd", sheet_name=None)
    results = {}
    for hotel, df_ in dfs.items():
        try:
            results[hotel] = _hotel_summary(df_)
        except Exception:
            results[hotel] = f"Error: Can't find summary data for {hotel} sheet"
    chats = {}
    for hotel, df_ in dfs.items():
        try:
            chats[hotel] = _chat_summary(df_)
        except Exception:
            chats[hotel] = f"Error: Can't find chat data for {hotel} sheet"

    return render_template("result.html", results=results, chats=chats)

if __name__ == "__main__":
    app.run(debug=True)


import requests
from lxml import etree
import time
from pyquery import PyQuery as pq

headers = {
        "User-Agent": "请复制自己浏览器"
    }
def get_Source(url,headers):
    r = requests.post(url, headers=headers)
    r.encoding = 'utf8'
    html = r.text
    return html

def jc(text,url):
    doc = pq(text)
    for td in doc("title").items():
        if td.text() == "解除屏蔽":
            print("被屏蔽，解除ing")
            from Agricultural.shangwu.solve.jiechu import Jiechu
            if Jiechu().solve("https://price.21food.cn/guoshu/shuiguo/"):
                return get_Source(url,headers)
            else:
                Jiechu().solve("https://price.21food.cn/guoshu/shuiguo/")
                return get_Source(url, headers)
        else:
            return text
def parse(url,crawl):
    for i in url:
        text = jc(get_Source(i,headers),i)
        dom = etree.HTML(text)
        path = "/html/body/div[2]/div[3]/div/div[2]/div[1]/div[2]/div[2]/ul/*"
        ul_text = dom.xpath(path)
        for li in ul_text:
            name = li.xpath('./table/tr/td[1]/a/text()')[0]
            market_name = li.xpath('./table/tr/td[2]/a/text()')[0]
            max = li.xpath('./table/tr/td[3]/span/text()')[0]
            min = li.xpath('./table/tr/td[4]/span/text()')[0]
            avg = li.xpath('./table/tr/td[5]/span/text()')[0]
            report = li.xpath('./table/tr/td[6]/span/text()')[0]
            if report == crawl:
                print(name, market_name, max, min, avg,report,crawl)
            else:
                break
        time.sleep(5)
    return False
#
# text = get_Source("https://price.21food.cn/guoshu/",headers)
# parse(text,"2021-03-06")


"""
This is a factory class dedicated to creating new instances of objects in
the Massenergize Database
"""

from _main_.utils.utils import get_models_and_field_types
from database import models
from _main_.utils.constants import CREATE_ERROR_MSG
from collections.abc import Iterable
from sentry_sdk import capture_message

MODELS_AND_FIELDS = get_models_and_field_types(models)

class CreateFactory:
  def __init__(self, name=None):
    self.name = name

  def verify_required_fields(self, model, args):
    errors = []
    required_fields = MODELS_AND_FIELDS[model]['required_fields']
    for f in required_fields:
      if f not in args and f !='id':
        errors.append(f"You are missing a required field: {f}")
    return errors

  def create(self, model, args={}):
    args.pop('id', None) #remove id provided.  We want db to assign its own
    errors = self.verify_required_fields(model, args)
    new_object = None

    if errors:
      return None, errors

    #if code gets here we have everything all required fields
    try:

      many_to_many_fields =  MODELS_AND_FIELDS[model]['m2m']
      fk_fields = MODELS_AND_FIELDS[model]['fk']
      field_values = {}
      for field_name, value in args.items():

        if field_name in fk_fields:
          fkModel = model._meta.get_field(field_name).remote_field.model
          field_values[field_name] = fkModel.objects.filter(pk=value).first()
        elif field_name not in many_to_many_fields:
          field_values[field_name] = value

      new_object = model.objects.create(**field_values)
      new_object.full_clean()
      new_object.save()
      for field_name, value in args.items():
        if field_name in many_to_many_fields:
          m2mModel = model._meta.get_field(field_name).remote_field.model
          if not isinstance(value, str) and isinstance(value, Iterable):
            addManyFunction = getattr(getattr(new_object, field_name), "set")
            addManyFunction(m2mModel.objects.filter(pk__in=value))
          else:
            addManyFunction = getattr(getattr(new_object, field_name), "set")
            addManyFunction(m2mModel.objects.filter(pk=value))

      new_object.save()

    except Exception as e:
      capture_message(str(e), level="error")
      errors =  [CREATE_ERROR_MSG, str(e)]
    return new_object, errors

  def update(self, model, args={}):

    errors = []
    new_object = None

    #if code gets here we have everything all required fields
    try:
      many_to_many_fields =  MODELS_AND_FIELDS[model]['m2m']
      fk_fields = MODELS_AND_FIELDS[model]['fk']

      id = args.pop('id', None)
      obj = model.objects.filter(pk=id).first()
      if not obj:
        return None, [f"Resource with id: {id} does not exist"]
      for field_name, value in args.items():
        if field_name in fk_fields:
          fkModel = model._meta.get_field(field_name).remote_field.model
          setattr(obj, field_name, fkModel.objects.filter(pk=value).first())
        elif field_name not in many_to_many_fields:
          setattr(obj, field_name, value)
        elif field_name in many_to_many_fields:
          m2mModel = model._meta.get_field(field_name).remote_field.model
          if not isinstance(value, str) and isinstance(value, Iterable):
            addManyFunction = getattr(getattr(obj, field_name), "set")
            addManyFunction(m2mModel.objects.filter(pk__in=value))
          else:
            addManyFunction = getattr(getattr(obj, field_name), "set")
            oldList = list(getattr(obj, field_name).all())
            res = oldList + list(m2mModel.objects.filter(pk=value))
            addManyFunction(res)

      obj.save()

    except Exception as e:
      capture_message(str(e), level="error")
      obj, errors =  None, [CREATE_ERROR_MSG, str(e)]

    return obj, errors


import pygame

window = pygame.display.set_mode((500,500))
red = (200,0,0)
rect = (70, 90, 100, 150) # x,y,width,height
rect2 = (335, 90, 100, 150) # x,y,width,height

active = True

while active:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            active = False
    pygame.draw.ellipse(window, red, rect) # DRAW ELLIPSE
    pygame.draw.ellipse(window, red, rect2)
    pygame.display.update()


"""
Defines an endpoint for gradebook data related to a course.
"""
import copy
import logging
import json
from collections import namedtuple
from contextlib import contextmanager

from django.conf import settings
from django.contrib.auth.models import User
from django.http import JsonResponse
from django.urls import reverse
from django.utils.translation import ugettext as _
from django.utils import timezone
from django.views.decorators.http import require_POST
from rest_framework import status
from rest_framework.response import Response
from six import text_type
from util.date_utils import to_timestamp

from courseware.courses import get_course_by_id
from lms.djangoapps.grades.api.v1.utils import (
    USER_MODEL,
    GradeViewMixin,
    PaginatedAPIView,
    verify_course_exists
)
from lms.djangoapps.grades.constants import ScoreDatabaseTableEnum
from lms.djangoapps.grades.course_data import CourseData
from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
from lms.djangoapps.grades.events import SUBSECTION_GRADE_CALCULATED, subsection_grade_calculated
from lms.djangoapps.grades.models import (
    PersistentCourseGrade,
    PersistentSubsectionGrade,
    PersistentSubsectionGradeOverride,
    PersistentSubsectionGradeOverrideHistory,
)
from lms.djangoapps.grades.services import GradesService
from lms.djangoapps.grades.signals.signals import GRADE_EDITED
from lms.djangoapps.grades.subsection_grade import CreateSubsectionGrade
from lms.djangoapps.grades.tasks import (
    recalculate_subsection_grade_v3,
    send_grade_override_email,
    update_course_progress
)
from lms.djangoapps.instructor.enrollment import get_user_email_language
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, UsageKey
from openedx.core.djangoapps.course_groups import cohorts
from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
from openedx.core.djangoapps.util.forms import to_bool
from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes
from student.auth import has_course_author_access
from student.models import CourseEnrollment
from student.roles import BulkRoleCache
from track.event_transaction_utils import (
    create_new_event_transaction_id,
    get_event_transaction_id,
    get_event_transaction_type,
    set_event_transaction_type
)

log = logging.getLogger(__name__)

@contextmanager
def bulk_gradebook_view_context(course_key, users):
    """
    Prefetches all course and subsection grades in the given course for the given
    list of users, also, fetch all the score relavant data,
    storing the result in a RequestCache and deleting grades on context exit.
    """
    PersistentSubsectionGrade.prefetch(course_key, users)
    PersistentCourseGrade.prefetch(course_key, users)
    CourseEnrollment.bulk_fetch_enrollment_states(users, course_key)
    cohorts.bulk_cache_cohorts(course_key, users)
    BulkRoleCache.prefetch(users)
    yield
    PersistentSubsectionGrade.clear_prefetched_data(course_key)
    PersistentCourseGrade.clear_prefetched_data(course_key)

def get_bool_param(request, param_name, default):
    """
    Given a request, parameter name, and default value, returns
    either a boolean value or the default.
    """
    param_value = request.query_params.get(param_name, None)
    bool_value = to_bool(param_value)
    if bool_value is None:
        return default
    else:
        return bool_value

def course_author_access_required(view):
    """
    Ensure the user making the API request has course author access to the given course.

    This decorator parses the course_id parameter, checks course access, and passes
    the parsed course_key to the view as a parameter. It will raise a
    403 error if the user does not have author access.

    Usage::
        @course_author_access_required
        def my_view(request, course_key):
            # Some functionality ...
    """
    def _wrapper_view(self, request, course_id, *args, **kwargs):
        """
        Checks for course author access for the given course by the requesting user.
        Calls the view function if has access, otherwise raises a 403.
        """
        course_key = CourseKey.from_string(course_id)
        if not has_course_author_access(request.user, course_key):
            raise DeveloperErrorViewMixin.api_error(
                status_code=status.HTTP_403_FORBIDDEN,
                developer_message='The requesting user does not have course author permissions.',
                error_code='user_permissions',
            )
        return view(self, request, course_key, *args, **kwargs)
    return _wrapper_view

def graded_subsections_for_course(course_structure):
    """
    Given a course block structure, yields the subsections of the course that are graded.
    Args:
        course_structure: A course structure object.  Not user-specific.
    """
    for chapter_key in course_structure.get_children(course_structure.root_block_usage_key):
        for subsection_key in course_structure.get_children(chapter_key):
            subsection = course_structure[subsection_key]
            if subsection.graded:
                yield subsection

GradebookUpdateResponseItem = namedtuple('GradebookUpdateResponseItem',
                                         ['user_id', 'usage_id', 'success', 'reason', 'grade_data'])

@view_auth_classes()
class GradebookBulkUpdateView(GradeViewMixin, PaginatedAPIView):
    """
    **Use Case**
        Creates `PersistentSubsectionGradeOverride` objects for multiple (user_id, usage_id)
        pairs in a given course, and invokes a Django signal to update subsection grades in
        an asynchronous celery task.

    **Example Request**
        POST /api/grades/v1/gradebook/{course_id}/bulk-update

    **POST Parameters**
        This endpoint does not accept any URL parameters.

    **Example POST Data**
          {
            9: {"block-v1:edX+DemoX+Demo_Course+type@sequential+block@basic_questions": 50,
                "block-v1:edX+DemoX+Demo_Course+type@sequential+block@advanced_questions": 100},

            11: {"block-v1:edX+DemoX+Demo_Course+type@sequential+block@basic_questions": 90}
          }

    **POST Response Values**
        An HTTP 202 may be returned if a grade override was created for each of the requested (user_id, usage_id)
        pairs in the request data.
        An HTTP 403 may be returned if the `writable_gradebook` feature is not
        enabled for this course.
        An HTTP 404 may be returned for the following reasons:
            * The requested course_key is invalid.
            * No course corresponding to the requested key exists.
            * The requesting user is not enrolled in the requested course.
        An HTTP 422 may be returned if any of the requested (user_id, usage_id) pairs
        did not have a grade override created due to some exception.  A `reason` detailing the exception
        is provided with each response item.

    **Example successful POST Response**
        [
          {
            "user_id": 9,
            "usage_id": ["some-requested-usage-id", "some-requested-usage-id"]
            "success": true,
            "reason": null,
            "grade_data": [
                {'category': 'Total', 'override': True, 'usage_key': '', 'grade': 75, 'class': 'high-column',
                    'percent': 0.75, 'detail': u'Total', 'label': 'Total'},
                {'category': u'Homework', 'override': True, 'usage_key': u'block-v1:edX+DemoX+Demo_Course+type@se..},
                {'category': u'Homework', 'override': True, 'usage_key': u'block-v1:edX+DemoX+Demo_Course+type@se..},
                {'category': u'Homework', 'override': False, 'usage_key': '', 'grade': 0, 'class': '', 'percent':..},
                {'category': u'Exam', 'override': False, 'usage_key': u'block-v1:edX+DemoX+Demo_Course+type@seq..}
            ]
          }
        ]
    """

    @verify_course_exists
    @course_author_access_required
    def post(self, request, course_key):
        """
        Creates or updates `PersistentSubsectionGradeOverrides` for the (user_id, usage_key)
        specified in the request data.  The `GRADE_EDITED` signal is invoked
        after the grade override is created, which triggers a celery task to update the
        course and subsection grades for the specified user.
        """

        course = get_course_by_id(course_key, depth=None)

        result = []
        email_list = []
        course_progress_users = []
        whole = False
        data = request.data['log']
        number_of_sections = int(request.data['number_of_sections'])
        import json
        data = json.loads(data)
        for user_id, keys in data.items():
            requested_user_id = int(user_id)
            usage_ids = list(set(keys))
            override_sections = {}
            if number_of_sections == len(usage_ids):
                whole = True
            try:
                user = self._get_single_user(request, course_key, requested_user_id)
            except USER_MODEL.DoesNotExist:
                continue

            for requested_usage_id in copy.copy(usage_ids):
                try:
                    usage_key = UsageKey.from_string(requested_usage_id)

                except InvalidKeyError as exc:
                    self._log_update_result(
                        request.user, requested_user_id, requested_usage_id, success=False, reason=text_type(exc)
                    )
                    usage_ids.remove(requested_usage_id)
                    continue

                try:
                    subsection_grade_model = PersistentSubsectionGrade.objects.get(
                        user_id=requested_user_id,
                        course_id=course_key,
                        usage_key=usage_key
                    )

                except PersistentSubsectionGrade.DoesNotExist:
                    subsection = course.get_child(usage_key)
                    if subsection:
                        subsection_grade_model = self._create_subsection_grade(user, course, subsection)
                    else:
                        self._log_update_result(
                            request.user, requested_user_id, requested_usage_id, success=False,
                            reason=u'usage_key does not exist in this course.'
                        )
                        usage_ids.remove(requested_usage_id)
                        continue

                if subsection_grade_model:
                    # if the current score equals to the override score, skip and remove the usage_id from the list
                    # This won't happen if we edit the score of subsections one by one, but it could happen when we
                    # edit the total score or the average score, because we have to change all the score of the included
                    # subsection to the same
                    override_score = keys[requested_usage_id] * subsection_grade_model.possible_all / 100
                    if override_score == subsection_grade_model.earned_all:
                        usage_ids.remove(requested_usage_id)
                        continue
                    else:
                        override_data = {
                            "earned_all_override": override_score,
                            "possible_all_override": subsection_grade_model.possible_all,
                            "earned_graded_override": override_score,
                            "possible_graded_override": subsection_grade_model.possible_all
                        }
                        override = self._create_override(request.user, subsection_grade_model, **override_data)
                        self._log_update_result(
                            request.user, requested_user_id, requested_usage_id, subsection_grade_model, override, True
                        )

            if usage_ids:
                course_grade = CourseGradeFactory().read(user, course)
                grade_summary = course_grade.summary
                graded_subsections = course_grade.graded_subsections_by_format
                section_breakdown = grade_summary['section_breakdown']
                for section in section_breakdown:
                    sections_by_format = graded_subsections.get(section.get('category'))
                    if sections_by_format:
                        pair = sections_by_format.popitem(last=False)
                        section['usage_key'] = unicode(pair[0])
                        if unicode(pair[0]) in usage_ids:
                            override_sections[pair[1].display_name] = int(section['percent'] * 100)
                        if pair[1].override is not None:
                            section['override'] = True
                            grade_summary['override'] = True
                    else:
                        continue
                grade_data = self._grade_data(grade_summary)
                result.append(GradebookUpdateResponseItem(
                    user_id=user.id,
                    usage_id=usage_ids,
                    success=True,
                    reason=None,
                    grade_data=grade_data
                ))
                email_list.append({
                    'total': int(grade_summary['percent'] * 100),
                    'name': user.profile.name or user.username,
                    'email': user.email,
                    'whole': whole,
                    'sections': override_sections,
                    'language': get_user_email_language(user)
                })
                course_progress_users.append(user.id)

        if email_list:
            # we don't finish the anaytics transcript feature yet, so use this fake link
            transcript_link = request.build_absolute_uri(reverse('analytics_my_transcript'))
            course_name = course.display_name_with_default_escaped
            platform_name = configuration_helpers.get_value('PLATFORM_NAME', settings.PLATFORM_NAME)
            email_service_enabled = configuration_helpers.get_value('ENABLE_EMAIL_SERVICE', True)
            from_alias = configuration_helpers.get_value('email_from_alias', settings.DEFAULT_FROM_EMAIL_ALIAS)
            from_address = configuration_helpers.get_value('email_from_address', settings.DEFAULT_FROM_EMAIL)
            from_address = "{0} <{1}>".format(from_alias, from_address)
            send_grade_override_email.apply_async(
                kwargs={
                    'student_info': email_list,
                    'transcript_link': transcript_link,
                    'course_name': course_name,
                    'platform_name': platform_name,
                    'email_service_enabled': email_service_enabled,
                    'from_address': from_address
                }
            )
            if course_progress_users:
                update_course_progress.apply_async(
                    kwargs={
                        'users': course_progress_users,
                        'course_id': unicode(course_key)
                    }
                )

        status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
        if len(result) > 0:
            status_code = status.HTTP_202_ACCEPTED

        return Response(
            {item.user_id: item._asdict() for item in result},
            status=status_code,
            content_type='application/json'
        )

    def _create_subsection_grade(self, user, course, subsection):
        course_data = CourseData(user, course=course)
        subsection_grade = CreateSubsectionGrade(subsection, course_data.structure, {}, {})
        return subsection_grade.update_or_create_model(user, force_update_subsections=True)

    def _create_override(self, request_user, subsection_grade_model, **override_data):
        """
        Helper method to create a `PersistentSubsectionGradeOverride` object
        and send a `GRADE_EDITED` signal.
        """
        override = PersistentSubsectionGradeOverride.update_or_create_override(
            requesting_user=request_user,
            subsection_grade_model=subsection_grade_model,
            feature=PersistentSubsectionGradeOverrideHistory.GRADEBOOK,
            **override_data
        )

        GRADE_EDITED.send(
            sender=None,
            user_id=subsection_grade_model.user_id,
            course_id=subsection_grade_model.course_id,
            modified=override.modified,
        )

        set_event_transaction_type(SUBSECTION_GRADE_CALCULATED)
        create_new_event_transaction_id()

        recalculate_subsection_grade_v3.apply(
            kwargs=dict(
                user_id=subsection_grade_model.user_id,
                anonymous_user_id=None,
                course_id=text_type(subsection_grade_model.course_id),
                usage_id=text_type(subsection_grade_model.usage_key),
                only_if_higher=False,
                expected_modified_time=to_timestamp(override.modified),
                score_deleted=False,
                event_transaction_id=unicode(get_event_transaction_id()),
                event_transaction_type=unicode(get_event_transaction_type()),
                score_db_table=ScoreDatabaseTableEnum.overrides,
                force_update_subsections=True,
            )
        )
        # Emit events to let our tracking system to know we updated subsection grade
        subsection_grade_calculated(subsection_grade_model)
        return override

    def _grade_data(self, grade_summary):
        data = []
        for section in grade_summary['section_breakdown']:
            attrs = {'percent': section['percent'], 'category': section['category'],
                     'label': section['label'], 'detail': section['detail'],
                     'class': '', 'grade': int(round(section['percent'] * 100)),
                     'usage_key': section.get('usage_key', ''),
                     'override': section.get('override', False)}
            if 'Avg' in section['label']:
                attrs['class'] = 'high-column'
            if section.get('override', False):
                attrs['override'] = True
            data.append(attrs)
        total = {'percent': grade_summary['percent'], 'category': 'Total', 'label': 'Total',
                 'detail': _('Total'),'class': 'high-column', 'grade': int(round(grade_summary['percent'] * 100)),
                 'override': grade_summary.get('override', False), 'usage_key': ''}
        if grade_summary.get('manually_changed'):
            total['manually_changed'] = True
        data = [total] + data
        return data

    @staticmethod
    def _log_update_result(
        request_user,
        user_id, usage_id,
        subsection_grade_model=None,
        subsection_grade_override=None,
        success=False,
        reason=None,
    ):

        log.info(
            u'Grades: Bulk_Update, UpdatedByUser: %s, User: %s, Usage: %s, Grade: %s, GradeOverride: %s, Success: %s, '
            u'Reason: %s',
            request_user.id,
            user_id,
            usage_id,
            subsection_grade_model,
            subsection_grade_override,
            success,
            reason
        )

@require_POST
def undo_override_for_student(request, course_id):
    user_id = int(request.POST.get('user_id'))
    user = User.objects.get(id=user_id)
    course_key = CourseKey.from_string(course_id)
    usage_ids = json.loads(request.POST.get('usage_ids'))
    for key in usage_ids:
        GradesService().undo_override_subsection_grade(user_id, course_id, key)
    GRADE_EDITED.send(
        sender=None,
        user_id=user_id,
        course_id=course_key,
        modified=timezone.now(),
    )
    grade_summary = CourseGradeFactory().read(user, course_key=course_key).summary
    update_course_progress.apply_async(
        kwargs={
            'users': [user_id],
            'course_id': course_id
        }
    )
    data = []
    for section in grade_summary['section_breakdown']:
        attrs = {'percent': section['percent'], 'category': section['category'],
                 'label': section['label'], 'detail': section['detail'],
                 'class': '', 'grade': int(round(section['percent'] * 100)),
                 'usage_key': section.get('usage_key', ''),
                 'override': section.get('override', False)}
        if 'Avg' in section['label']:
            attrs['class'] = 'high-column'
        if section.get('override', False):
            attrs['override'] = True
        data.append(attrs)
    total = {'percent': grade_summary['percent'], 'category': 'Total', 'label': 'Total',
             'detail': _('Total'), 'class': 'high-column', 'grade': int(round(grade_summary['percent'] * 100)),
             'override': grade_summary.get('override', False), 'usage_key': ''}
    data = [total] + data
    return JsonResponse({user_id: data})


"""
Project: Twiptter
Author: iHazzu
Function: Collects posts from Twitter and publishes to a WordPress site
Date: 07/02/2022
"""

from datetime import timedelta, datetime
from twitter import search_tweets
from wordpress import publish_tweet
import logging

# This script should run every 10 minutes
now = datetime.utcnow()
last_search = now - timedelta(minutes=10)
tweets_list = []
try:
    tweets_list = search_tweets(last_search)
    for tweet in tweets_list:   # post tweets on Wordpress
        publish_tweet(tweet)
except Exception as error:
    logging.basicConfig(filename="logs/main_logs.log", level=logging.ERROR)
    logging.exception(f" {datetime.utcnow()} - EXCEPTION: {error}\nTWEETS: {tweets_list}")


"""Added smart_off_active

Revision ID: 455de5a3503e
Revises: eaa82f9ded8a
Create Date: 2020-11-07 16:43:36.205427

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '455de5a3503e'
down_revision = 'eaa82f9ded8a'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('light') as batch_op:
        batch_op.add_column(
            sa.Column('smart_off_active', sa.Boolean(), nullable=True)
        )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('light') as batch_op:
        batch_op.drop_column('smart_off_active')
    # ### end Alembic commands ###


import random
import sqlite3
import numpy as np

from cannier_framework.collate import collate_data_proj

def test_collate_data_proj_items(args, db_file):
    items_input = [
        ("test_a", 0, 10, 0,  10,  0, 0),
        ("test_b", 0, 10, 5,  10,  0, 1),
        ("test_c", 0, 10, 0,  10,  5, 0),
        ("test_d", 0,  9, 0,  10,  0, 1),
        ("test_e", 0, 10, 5,  10,  5, 0),
        ("test_f", 0, 10, 0,  10, 10, 1),
        ("test_g", 0, 10, 10, 10,  5, 0)
    ]

    random.shuffle(items_input)
    args.n_repeats = 0
    args.n_reruns = 10

    with sqlite3.connect(db_file) as con:
        cur = con.cursor()

        cur.execute(
            "replace into counters "
            "values (1, 1, 0, 10, 10)"
        )

        cur.executemany(
            "insert into item "
            "values (null, ?, ?, ?, ?, ?, ?, ?)",
            items_input
        )

        cur.execute(
            "select id, nodeid "
            "from item"
        )

        id_to_nodeid = {
            item_id: nodeid for item_id, nodeid in cur.fetchall()
        }

        id_map, items_output, *_ = collate_data_proj(cur, 0)

    nodeid_to_id = {id_to_nodeid[id1]: id2 for id1, id2 in id_map.items()}
    assert "test_d" not in nodeid_to_id

    order = [
        nodeid_to_id[nodeid] for nodeid in (
            "test_a", "test_b", "test_c", "test_e", "test_f", "test_g"
        )
    ]

    items_expected = np.array([
        [0, 0,  0, 0, 0, 0, 0],
        [0, 5,  0, 1, 0, 0, 0],
        [0, 0,  5, 0, 1, 1, 0],
        [0, 5,  5, 1, 0, 1, 0],
        [0, 0, 10, 0, 1, 1, 0],
        [0, 10, 5, 0, 1, 0, 0],
    ])

    assert (items_output[order] == items_expected).all()

def test_collate_data_proj_features(args, db_file):
    items = [
        ("test_a", 5, 10,  0, 10,  0, 0),
        ("test_b", 5, 10,  0, 10, 10, 1),
        ("test_c", 5,  9,  0, 10,  0, 1),
        ("test_d", 5, 10, 10, 10,  0, 0),
    ]

    features_input = [[i + 1] * 19 for i in range(len(items))] * 5
    random.shuffle(features_input)
    args.n_repeats = 5
    args.n_reruns = 10

    with sqlite3.connect(db_file) as con:
        cur = con.cursor()

        cur.execute(
            "replace into counters "
            "values (1, 1, 5, 10, 10)"
        )

        cur.executemany(
            "insert into item "
            "values (null, ?, ?, ?, ?, ?, ?, ?)",
            items
        )

        cur.executemany(
            "insert into features "
            "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            features_input
        )

        _, _, features_output, _ = collate_data_proj(cur, 0)

    features_expected = np.array(
        [item_id * np.ones((5, 18)) for item_id in (1, 2, 4)]
    )

    assert (features_output == features_expected).all()

def test_collate_data_proj_dependencies(args, db_file):
    items = [
        ("test_a", 0, 5, 0, 5, 0, 1),
        ("test_b", 0, 5, 0, 5, 5, 1),
        ("test_c", 0, 5, 5, 5, 0, 1),
        ("test_d", 0, 5, 0, 5, 2, 1),
    ]

    dependencies_input = [(2, 1), (3, 1), (4, 2)]
    random.shuffle(dependencies_input)
    args.n_repeats = 0
    args.n_reruns = 5

    with sqlite3.connect(db_file) as con:
        cur = con.cursor()

        cur.execute(
            "replace into counters "
            "values (1, 1, 0, 5, 5)"
        )

        cur.executemany(
            "insert into item "
            "values (null, ?, ?, ?, ?, ?, ?, ?)",
            items
        )

        cur.executemany(
            "insert into dependency "
            "values (?, ?)",
            dependencies_input
        )

        _, _, _, dependencies_output = collate_data_proj(con.cursor(), 0)

    dependencies_output = np.unpackbits(dependencies_output, axis=0, count=4)

    dependencies_expected = np.array([
        [0, 0, 0, 0],
        [1, 0, 0, 0],
        [1, 0, 0, 0],
        [0, 1, 0, 0]
    ])

    assert (dependencies_output == dependencies_expected).all()


from SW_GPTS_Learner import *
from ThreePhasesScenario import *
from Optimizer import *

from tqdm import tqdm
import math
import numpy as np
from matplotlib import pyplot as plt

def compute_ideal(scenarios):
    # Compute the real value of the tru function
    real_values = []
    for scenario in scenarios:
        real_values.append(scenario.phase_values[:number_of_budgets])

    # FIND IDEAL ARMS (to compute regret)
    # Given the real value of the true function, find the ideal combination of arms
    arms = ideal_optimizer.optimize(real_values)

    # Compute the ideal optimal result
    result = 0
    for index, scenario in enumerate(scenarios):
        result += scenario.phase_values[arms[index]]

    return arms, result

number_of_experiments = 5
window_coefficients = [1, 3, 5, 0]

# The budget cap and the possible daily budget that can be set
budget_cap = 27000
daily_budgets = np.linspace(5000, budget_cap, endpoint=True, num=45).astype(int)

# The number of possible budgets that can be allocated to each subcampaign
number_of_budgets = 11

time_horizon = 300
phase_duration = int(time_horizon / 3)

regret = []

fig, ax = plt.subplots(2, 2, figsize=(24, 18))

for i in range(0, len(window_coefficients)):

    # See slide 67 1-09
    window_coefficient = window_coefficients[i]
    window_length = int(window_coefficient * math.sqrt(time_horizon))

    for e in range(0, number_of_experiments):

        print('\n')
        print('Starting experiment', e + 1, '(c=', window_coefficients[i], ')')

        scenarios = [ThreePhasesScenario(daily_budgets=daily_budgets,
                                         campaign=0,
                                         phase_duration=phase_duration),
                     ThreePhasesScenario(daily_budgets=daily_budgets,
                                         campaign=1,
                                         phase_duration=phase_duration),
                     ThreePhasesScenario(daily_budgets=daily_budgets,
                                         campaign=2,
                                         phase_duration=phase_duration)]

        subcampaigns = [SW_GPTS_Learner(arms=daily_budgets[:number_of_budgets],
                                        window_length=window_length),
                        SW_GPTS_Learner(arms=daily_budgets[:number_of_budgets],
                                        window_length=window_length),
                        SW_GPTS_Learner(arms=daily_budgets[:number_of_budgets],
                                        window_length=window_length)]

        optimizer = Optimizer(daily_budgets, len(daily_budgets))
        ideal_optimizer = Optimizer(daily_budgets, len(daily_budgets))

        # The optimal and the ideal result obtained by playing the best possible combination of arms known
        optimal_rewards_per_round = np.zeros(time_horizon)
        ideal_rewards_per_round = np.zeros(time_horizon)

        ideal_arms, ideal_result = compute_ideal(scenarios)

        for t in tqdm(range(0, time_horizon)):

            # The ideal result obtainable at time t
            if t % phase_duration == 1:
                ideal_arms, ideal_result = compute_ideal(scenarios)

            # Sample the Gaussian Process for each subcampaign
            sampled_values = []
            for subcampaign in subcampaigns:
                sampled_values.append(subcampaign.sample_values())

            # FIND OPTIMAL ARMS
            # Find the best combination of arms to play at time t based on the sampling of the Gaussian Processes
            optimal_arms = optimizer.optimize(sampled_values)
            # The optimal result obtainable at time t by playing the optimal arms
            optimal_result = 0

            for sub_index in range(0, len(subcampaigns)):
                # Sample the true function to get a reward for the played arm
                reward = scenarios[sub_index].play_round(optimal_arms[sub_index])
                optimal_result += reward

                # Update the model based on the result of the played arm
                subcampaigns[sub_index].update(optimal_arms[sub_index], reward)

                # Advance the time of the scenario and eventually trigger a new phase
                scenarios[sub_index].advance_time()

            optimal_rewards_per_round[t] = optimal_result
            ideal_rewards_per_round[t] = ideal_result
            # print("\n",
            #       "[", t, "]Optimal solution:", optimal_arms, "(", ideal_arms, ")",
            #       "with a result of: ", round(optimal_result, 2), " (", round(ideal_result, 2), ").")

        regret.append(ideal_rewards_per_round - optimal_rewards_per_round)
        print("\n", "Regret:", np.cumsum(ideal_rewards_per_round - optimal_rewards_per_round)[-1], "\n")

    # PLOTTING THE REGRET

    ax[int(i/2), i % 2].set_title("Regret for c = " + str(window_coefficient), fontsize=17, ha='center')

    ax[int(i/2), i % 2].grid(axis='x', alpha=0.3, linestyle='--')

    ax[int(i/2), i % 2].set_ylim(bottom=0)
    ax[int(i/2), i % 2].set_xlim(left=0)

    ax[int(i/2), i % 2].set_xlabel("Day", fontsize=14)
    ax[int(i/2), i % 2].set_ylabel("Regret", fontsize=14)

    ax[int(i/2), i % 2].tick_params(length=0)
    ax[int(i/2), i % 2].set_yticks(np.linspace(0, 40000, 11))
    ax[int(i/2), i % 2].set_yticklabels(np.linspace(0, 40000, 11).astype(np.int64), fontsize=12, alpha=0.7)

    ax[int(i/2), i % 2].spines['right'].set_alpha(0)
    ax[int(i/2), i % 2].spines['left'].set_alpha(0.3)
    ax[int(i/2), i % 2].spines['top'].set_alpha(0)
    ax[int(i/2), i % 2].spines['bottom'].set_alpha(0.3)

    ax[int(i/2), i % 2].axvline(phase_duration*1, color='lightcoral')
    ax[int(i/2), i % 2].axvline(phase_duration*2, color='lightcoral')

    ax[int(i/2), i % 2].plot(np.cumsum(np.mean(regret, axis=0)), linewidth='2')
    ax[int(i/2), i % 2].set_xlim([0, time_horizon])

    # ax[int(i/2), i % 2].legend(frameon=False)

fig.savefig('chapter3_regret.png')

fig.show()


import cv2, time, pandas
from datetime import datetime
import imutils

first_frame = None
status_list = [None, None]  # to prevent index out of bound error at the beginning
times = []
df = pandas.DataFrame(columns=["Start", "End"])

# video = cv2.VideoCapture(cv2.CAP_DSHOW)
f = "e:\\DEV\\MSVC_Projects\\Astronomy\\test.mp4"
video = cv2.VideoCapture(f)

time.sleep(2)
is_video = True
vs = video
firstFrame = None
frame_num = 0
dx = 127
dy = 255
while True:
    # grab the current frame and initialize the occupied/unoccupied
    # text
    frame = vs.read()
    frame = frame if not is_video else frame[1]

    # if the frame could not be grabbed, then we have reached the end
    # of the video
    if frame is None:
        break

    # resize the frame, convert it to grayscale, and blur it
    frame = imutils.resize(frame, width=700)
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    gray = cv2.GaussianBlur(gray, (1, 1), 0)

    # if the first frame is None, initialize it
    if firstFrame is None:
        firstFrame = gray
        continue

    # compute the absolute difference between the current frame and
    # first frame
    frameDelta = cv2.absdiff(firstFrame, gray)
    thresh = cv2.threshold(frameDelta, dx, dy, cv2.THRESH_BINARY)[1]
    firstFrame = gray

    # dilate the thresholded image to fill in holes, then find contours
    # on thresholded image
    thresh = cv2.dilate(thresh, None, iterations=2)
    cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                            cv2.CHAIN_APPROX_SIMPLE)
    cnts = imutils.grab_contours(cnts)

    # loop over the contours
    for c in cnts:
        # if the contour is too small, ignore it
        # if cv2.contourArea(c) < args["min_area"]:
        t = cv2.contourArea(c)
        if t < 1:
            continue
        if t > 200:
            continue

        # compute the bounding box for the contour, draw it on the frame,
        # and update the text
        (x, y, w, h) = cv2.boundingRect(c)
        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)

    cv2.putText(frame, "frame {}".format(frame_num), (0, 20),
                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
    cv2.putText(frame, "DX={0} DY={1}".format(dx, dy), (0, 40),
                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)

    # show the frame and record if the user presses a key
    cv2.imshow("Security Feed", frame)
    cv2.moveWindow("Security Feed", 0, 0)
    cv2.imshow("Frame Delta", frameDelta)
    cv2.moveWindow("Frame Delta", 0, 400)
    cv2.imshow("Thresh", thresh)
    cv2.moveWindow("Thresh", 0, 800)
    key = cv2.waitKeyEx(1)

    if key in [0x1b, ord('q')]:
        break
    if key == 0x250000:  # left
        frame_num -= 30
        if (frame_num < 0):
            frame_num = 0
        print("rewind")
        vs.set(cv2.CAP_PROP_POS_FRAMES, frame_num)
    if key == 0x260000:  # up
        dx += 1
        if (dx > 255):
            dx = 255
    if key == 0x280000:  # down
        dx -= 1
        if (dx < 0):
            dx = 0
    if key == 0x2d:  # -
        dy -= 10
        if (dy < 0):
            dy = 0
    if key == 0x2b:  # +
        dy += 10
        if (dy > 255):
            dy = 255
        pass
    if key >= 0:
        print(hex(key))

    frame_num += 1
    if(frame_num>300):
        frame_num=0
        print("rewind")
        vs.set(cv2.CAP_PROP_POS_FRAMES, frame_num)

print(status_list)
print(times)

video.release()
cv2.destroyAllWindows()


from django.contrib.auth import get_user_model
from rest_framework import serializers
from ..models import Subreddit, Post, PostVotes, PostComment

User = get_user_model()

class PostSerializer(serializers.ModelSerializer):
    subreddit = serializers.PrimaryKeyRelatedField(
        queryset=Subreddit.objects.all()
    )

    class Meta:
        model = Post
        fields = [
            "id",
            "created",
            "modified",
            'time_since_post',
            "title",
            "link",
            "description",
            "description_br",
            "owner",
            'score',
            "username",
            "owner_url",
            "subreddit",
            "subreddit_name",
            'slug',
            'subreddit_slug',
            'full_url',
            'user_vote',
        ]
        extra_kwargs = {
            "url": {"view_name": "api:posts", "lookup_field": "title"}
        }
        # extra_kwargs = {
        #     "url": {"view_name": "api:user-detail", "lookup_field": "username"}
        # }

class PostVotesSerializer(serializers.ModelSerializer):

    class Meta:
        model = PostVotes
        fields = ['post_id', 'user_id', 'vote', 'id',
                  ]

class AllPostsSerializer(serializers.ModelSerializer):
    subreddit = serializers.PrimaryKeyRelatedField(
        queryset=Subreddit.objects.all()
    )
    queryset = Post.objects.all()

    class Meta:
        model = Post
        fields = [
            "id",
            "created",
            "modified",
            'time_since_post',
            "title",
            "link",
            "description",
            "description_br",
            "owner",
            "username",
            "owner_url",
            "subreddit",
            "subreddit_name",
            'slug',
            'subreddit_slug',
            'score',
            'full_url',
            'user_vote',
            'user_up_style',
            'user_down_style',
            'weighted_score',
            'age_in_days',
            'number_of_comments',
        ]
        extra_kwargs = {
            "url": {"view_name": "api:allposts", "lookup_field": "title"}
        }
        # extra_kwargs = {
        #     "url": {"view_name": "api:user-detail", "lookup_field": "username"}
        # }

class CommentSerializer(serializers.ModelSerializer):
    children = serializers.SerializerMethodField(
        read_only=True, method_name="get_children")

    post_id = serializers.PrimaryKeyRelatedField(
        queryset=Post.objects.all()
    )
    user_id = serializers.HiddenField(default=serializers.CurrentUserDefault())

    parent = serializers.PrimaryKeyRelatedField(
        queryset=PostComment.objects.all(), allow_null=True)

    class Meta:
        model = PostComment
        fields = [
            "id",
            "post_id",
            "created",
            "modified",
            'time_since_comment',
            "comment",
            "comment_br",
            "user_id",
            "username",
            "owner_url",
            'score',
            'user_vote',
            'user_up_style',
            'user_down_style',
            'level',
            'parent',
            'children',
            'tldr',
        ]

        extra_kwargs = {
            "url": {"view_name": "api:allposts", "lookup_field": "title"}
        }

    def get_children(self, obj):
        """ self referral field """
        serializer = CommentSerializer(
            instance=obj.comment_reply.all(),
            many=True
        )
        return serializer.data


import math
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm

def main():
    line = input()
    i = 0

    vs = []

    indeces = []
    noError = True
    while(noError):
        x = float(line)
        vs.append(x)
        indeces.append(i)
        i = i+1

        try:
            line= input()
        except EOFError as e:
            noError = False

    #now we wnat to compute our stats. we want to first get the different size of jumps in an array...
    accels = []
    maxaccel = 0

    for j in range(len(indeces)-1):
        accel =vs[j]-vs[j+1]
        accels.append(accel)
        if(accel>maxaccel):
            maxaccel = accel
    #and we also want to know whhat the likelihood of one being negative is...
    """#now we want ot get the types of jumps...
    smallJMP = 0.008131838385252443
    medsJMP = 1/150
    medlJMP = 2/210
    largeJMP = 0.01
    #s, ms, ml, l
    dist = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.010]
    counts = [0,0,0,0,0,0,0,0,0,0]
    bestdistance = 100
    bestloc = 0
    for accel in accels:
        bestdistance = 100
        bestloc = 0
        for d in range(0,len(dist)):
            distance = abs(accel-dist[d])
            if(distance<bestdistance):
                bestloc = d
                bestdistance = dist[d]
        counts[bestloc] = counts[bestloc]+1
    print(counts)
    """
    plt.hist(vs, 100)
    plt.show()

    """plt.plot(indeces, vs,
             label='vel', color='r', linewidth=0.3)

    plt.xlabel('Data Points', fontsize=20)
    plt.ylabel('vel', fontsize=20)
    plt.legend()
    plt.show()"""

    #now, let's get an average for each one of the histogram "groups". We have:
    mu,std = norm.fit(vs)
    print(mu)
    print(std)

if __name__ == '__main__':
    main()


import json
import logging
import os
import traceback
from optparse import make_option

from django.core.management.base import BaseCommand, CommandError

from instances.models import Instance

logger = logging.getLogger(__name__)

def _stats_pretty(stats):
    return ' '.join(["%s:%d" % (cls.__name__, n) for cls, n in stats.items()])

class ImportCommand(BaseCommand):

    importer_class = None
    document_extension = ''

    option_list = BaseCommand.option_list + (
        make_option('--commit', action='store_true', help='Whether to commit to the database or not'),
        make_option('--instance', action='store', help='Label of instance to add data to'),
        make_option('--file', action='store', help='document to import'),
        make_option('--dir', action='store', help='directory of documents to import'),
        make_option(
            '--no-verify', action='store_false', default=True, dest='verify',
            help='Whether to verify SSL certificates or not'),
        make_option(
            '--start-date', action='store', default='',
            help='earliest date to process, in yyyy-mm-dd format'),
        make_option(
            '--dump-users', action='store', default='',
            help='dump a json list to <file> (only valid with --dir for now)'),
        make_option(
            '--clobber-existing', action='store_const', const='replace', dest='clobber',
            help='Whether to replace sections with the same heading'),
        make_option(
            '--skip-existing', action='store_const', const='skip', dest='clobber',
            help='Whether to skip sections with the same heading'),
        make_option(
            '--merge-existing', action='store_const', const='merge', dest='clobber',
            help='Whether to merge sections with the same heading'),
        )

    def handle(self, *args, **options):
        verbosity = int(options['verbosity'])

        if options['commit']:
            if not options['instance']:
                raise CommandError("You must specify an instance")
            instance, __ = Instance.objects.get_or_create(label=options['instance'])
        else:
            instance = Instance(label=options['instance'])
        options['instance'] = instance

        if options['file']:
            filename = os.path.expanduser(options['file'])
            (stats, speakers) = self.import_document(filename, **options)
            if verbosity > 1 and stats:
                logger.info("Imported %s\n\n" % _stats_pretty(stats))
        elif options['dir']:
            files = sorted(self.document_list(options))

            if len(files):
                speakers = {}
                for f in files:
                    (stats, spkrs) = self.import_document(f, **options)
                    speakers.update(spkrs)

                    if verbosity > 1 and stats:
                        logger.info("%s: Imported %s\n" % (f, _stats_pretty(stats)))

                dump_users = os.path.expanduser(options['dump_users'])
                if dump_users:
                    out = open(dump_users, 'w')
                    speakers_list = [(k, speakers[k]) for k in speakers]
                    out.write(json.dumps(speakers_list, indent=4))

                    if verbosity > 1:
                        logger.info("Saved speakers list to %s\n" % dump_users)
            else:
                logger.info("No .%s files found in directory" % self.document_extension)
        else:
            logger.info(self.help)

    def document_list(self, options):
        dir = os.path.expanduser(options['dir'])

        start_date = options['start_date']
        if start_date:
            def valid(f):
                return f >= start_date
        else:
            def valid(f):
                return True

        return [os.path.join(root, filename)
                for (root, _, files)
                in os.walk(dir)
                for filename in files
                if filename[-4:] == '.%s' % self.document_extension and valid(filename)]

    def document_valid(self, path):
        return os.path.isfile(path)

    def import_document(self, path, **options):
        verbosity = int(options['verbosity'])

        if not self.document_valid(path):
            raise CommandError("No document found")

        if verbosity > 1:
            logger.info("Starting import: %s\n" % path)

        if self.importer_class is None:
            raise CommandError("No importer_class specified!")

        importer = self.importer_class(**options)

        try:
            importer.import_document(path)
        except Exception as e:
            logger.error("An exception of type %s occurred, arguments:\n%s\n%s" % (
                type(e).__name__, e, traceback.format_exc()))
            return (None, {})

        return (importer.stats, importer.speakers)


from queue import Queue
from threading import Thread
from threading import RLock
# from time import sleep
# from uuid import uuid4

import logging
# import pprint

from miniflow.canvas import (
    Task, Single, Group,
    AbstractChain, Chain, End
)

class Result:

    def __init__(self, task, value):
        self.task = task
        self.value = value

default_logger = logging.getLogger('canvas_executor')
log_handler = logging.StreamHandler()
log_handler.setLevel(logging.DEBUG)
FORMAT = '[%(levelname)s] %(asctime)-15s [%(threadName)s] %(message)s'
formatter = logging.Formatter(FORMAT)
log_handler.setFormatter(formatter)
default_logger.addHandler(log_handler)

class Service:

    def __init__(self, n_workers, logger=default_logger):

        self.logger = logger

        self.single_queue = Queue()
        self.result_queue = Queue()

        self.workers = []
        for i in range(n_workers):
            worker = Thread(target=self.work, args=(), name=f'Worker-{i+1}')
            self.workers.append(worker)

        self.handler = Thread(target=self.handle, args=(), name=f'Handler')

        self.sync_lock = RLock()
        self.sync_trigger = {}

        self.call_next_lock = RLock()
        self.call_next = {}

        self.stop_signal = None
        self.running_state_lock = RLock()
        self.running = 0

    def start(self):
        for worker in self.workers:
            worker.start()
        self.handler.start()

    def stop(self):
        for _ in self.workers:
            self.single_queue.put(self.stop_signal)
        self.result_queue.put(self.stop_signal)

    def submit_task(self, task: Task):

        def get_chain_end(chain: AbstractChain):
            if isinstance(chain, End):
                return chain
            elif isinstance(chain, Chain):
                return get_chain_end(chain.chain)
            else:
                raise

        if isinstance(task, End):
            end = task
            self.single_queue.put(end)
        elif isinstance(task, Single):
            single = task
            self.single_queue.put(single)
        else:
            assert isinstance(task, Group)
            group = task
            for chn in group.chains:
                assert isinstance(chn, Chain)
                end = get_chain_end(chn)
                with self.sync_lock:
                    self.sync_trigger[end] = group
            for chn in group.chains:
                self.submit_chain(chn)

    def submit_chain(self, chain: AbstractChain):
        if isinstance(chain, End):
            end = chain
            self.submit_task(end)
        else:
            assert isinstance(chain, Chain)
            task = chain.task
            assert isinstance(task, (Single, Group))
            downstream = chain.chain
            with self.call_next_lock:
                self.call_next[task] = downstream
            self.submit_task(task)

    def work(self):

        with self.running_state_lock:
            self.running += 1

        while True:
            # single_task can be either Single or End, or a stop signal for graceful stop.
            single_task = self.single_queue.get()

            if single_task is self.stop_signal:
                self.logger.debug('Exiting.')
                with self.running_state_lock:
                    self.running -= 1
                break

            try:
                value = single_task.run()
            except Exception as e:
                value = e
            result = Result(single_task, value)
            self.result_queue.put(result)

    def handle(self):
        while True:
            result = self.result_queue.get()

            if result is self.stop_signal:
                self.logger.debug('Exiting.')
                break

            if isinstance(result.task, End):
                # chain end indicates synchronization
                end = result.task
                self.logger.debug(f'Handling {end}')
                with self.sync_lock:
                    if end in self.sync_trigger:
                        group = self.sync_trigger.pop(end)
                        self.logger.debug(f'Triggering sync for {group}')
                        if group not in self.sync_trigger.values():
                            # group finished, make a group result
                            self.result_queue.put(Result(group, None))
            else:
                # normal result indicates chain continuation
                downstream = None
                with self.call_next_lock:
                    downstream = self.call_next.pop(result.task)
                if downstream:
                    self.logger.debug(f'Continuing chain {downstream}')
                    self.submit_chain(downstream)