"""Parser object that performs coarse-to-fine and postprocessing.

Additionally, a simple command line interface similar to bitpar."""
from __future__ import print_function
import io
import os
import re
import sys
import time
import gzip
import codecs
import logging
import tempfile
import traceback
import string  # pylint: disable=W0402
import multiprocessing
if sys.version[0] > '2':
	imap = map
else:
	from itertools import imap
from math import exp, log
from heapq import nlargest
from getopt import gnu_getopt, GetoptError
from operator import itemgetter
from functools import wraps
import numpy as np
from discodop import plcfrs, pcfg
from discodop.grammar import defaultparse
from discodop.containers import Grammar
from discodop.coarsetofine import prunechart, whitelistfromposteriors
from discodop.disambiguation import getderivations, marginalize, doprerank
from discodop.tree import Tree
from discodop.lexicon import replaceraretestwords, UNKNOWNWORDFUNC, UNK
from discodop.treebank import WRITERS, writetree
from discodop.treebanktransforms import reversetransform, rrbacktransform, \
		saveheads, NUMBERRE, readheadrules
from discodop.treetransforms import mergediscnodes, unbinarize, \
		removefanoutmarkers

USAGE = '''
usage: %(cmd)s [options] <grammar/> [input [output]]
or:    %(cmd)s --simple [options] <rules> <lexicon> [input [output]]

'grammar/' is a directory with a model produced by "discodop runexp".
When no filename is given, input is read from standard input and the results
are written to standard output. Input should contain one sentence per line
with space-delimited tokens. Output consists of bracketed trees in
selected format. Files must be encoded in UTF-8.

General options:
  -x             Input is one token per line, sentences separated by two
                 newlines (like bitpar).
  -b k           Return the k-best parses instead of just 1.
  --prob         Print probabilities as well as parse trees.
  --tags         Tokens are of the form "word/POS"; give both to parser.
  --fmt=(export|bracket|discbracket|alpino|conll|mst|wordpos)
                 Format of output [default: discbracket].
  --numproc=k    Launch k processes, to exploit multiple cores.
  --simple       Parse with a single grammar and input file; similar interface
                 to bitpar. The files 'rules' and 'lexicon' define a binarized
                 grammar in bitpar or PLCFRS format.
  --verbosity=x  0 <= x <= 4. Same effect as verbosity in parameter file.

Options for simple mode:
  -s x           Use "x" as start symbol instead of default "TOP".
  --bt=file      Apply backtransform table to recover TSG derivations.
  --obj=(mpd|mpp|mrp|mcc|shortest|sl-dop)
                 Objective function to maximize [default: mpd].
  -m x           Use x derivations to approximate objective functions;
                 mpd and shortest require only 1.
  --bitpar       Use bitpar to parse with an unbinarized grammar.
''' % dict(cmd=sys.argv[0], fmt=','.join(WRITERS))

DEFAULTSTAGE = dict(
		name='stage1',  # identifier, used for filenames
		mode='plcfrs',  # use the agenda-based PLCFRS parser
		prune=False,  # whether to use previous chart to prune this stage
		split=False,  # split disc. nodes VP_2[101] as { VP*[100], VP*[001] }
		splitprune=False,  # treat VP_2[101] as {VP*[100], VP*[001]} for pruning
		markorigin=False,  # mark origin of split nodes: VP_2 => {VP*1, VP*2}
		collapselabels=None,  # options: None, 'head', 'all'. TODO: implement.
		k=50,  # no. of coarse pcfg derivations to prune with; k=0: filter only
		dop=None,  # DOP mode (DOP reduction / double DOP)
		binarized=True,  # for double dop, whether to binarize extracted grammar
			# (False requires use of bitpar)
		sample=False, kbest=True,
		m=10,  # number of derivations to sample/enumerate
		estimator='rfe',  # choices: rfe, ewe
		objective='mpp',  # choices: mpp, mpd, shortest, sl-dop[-simple]
			# NB: w/shortest derivation, estimator only affects tie breaking.
		sldop_n=7,  # number of trees to consider when using sl-dop[-simple]
		mcc_labda=1.0,  # weight to assign to recall vs. mistake rate with mcc
		mcc_labels=None,  # optionally, set of labels to optimize for with mcc
		packedgraph=False,  # use packed graph encoding for DOP reduction
		iterate=False,  # for double dop, whether to add fragments of fragments
		complement=False,  # for double dop, whether to include fragments which
			# form the complement of the maximal recurring fragments extracted
		neverblockre=None,  # do not prune nodes with label that match regex
		estimates=None,  # compute, store & use outside estimates
		)

class DictObj(object):
	"""Trivial class to wrap a dictionary for reasons of syntactic sugar."""

	def __init__(self, *a, **kw):
		self.__dict__.update(*a, **kw)

	def update(self, *a, **kw):
		"""Update/add more attributes."""
		self.__dict__.update(*a, **kw)

	def __getattr__(self, name):
		"""Dummy function for suppressing pylint E1101 errors."""
		raise AttributeError('%r instance has no attribute %r' % (
				self.__class__.__name__, name))

	def __repr__(self):
		return '%s(%s)' % (self.__class__.__name__,
			',\n\t'.join('%s=%r' % a for a in self.__dict__.items()))

PARAMS = DictObj()  # used for multiprocessing when using CLI of this module

def main():
	"""Handle command line arguments."""
	flags = 'prob tags bitpar simple'.split()
	options = flags + 'obj= bt= numproc= fmt= verbosity='.split()
	try:
		opts, args = gnu_getopt(sys.argv[1:], 'b:s:m:x', options)
	except GetoptError as err:
		print(err, USAGE)
		return
	if not 1 <= len(args) <= 4:
		print('ERROR: incorrect number of arguments')
		print(USAGE)
		return
	for n, filename in enumerate(args):
		if not os.path.exists(filename):
			raise ValueError('file %d not found: %r' % (n + 1, filename))
	opts = dict(opts)
	numparses = int(opts.get('-b', 1))
	top = opts.get('-s', 'TOP')
	prob = '--prob' in opts
	tags = '--tags' in opts
	oneline = '-x' not in opts
	if '--simple' in opts:
		if not 2 <= len(args) <= 4:
			print('ERROR: incorrect number of arguments')
			print(USAGE)
			return
		rules = (gzip.open if args[0].endswith('.gz') else open)(args[0]).read()
		lexicon = codecs.getreader('utf-8')((gzip.open if args[1].endswith('.gz')
				else open)(args[1])).read()
		bitpar = rules[0] in string.digits
		if '--bitpar' in opts:
			if not bitpar:
				raise ValueError('bitpar requires bitpar grammar format.')
			mode = 'pcfg-bitpar-nbest'
		else:
			mode = 'pcfg' if bitpar else 'plcfrs'
		grammar = Grammar(rules, lexicon, start=top, bitpar=bitpar,
				binarized='--bitpar' not in opts)
		stages = []
		stage = DEFAULTSTAGE.copy()
		backtransform = None
		if opts.get('--bt'):
			backtransform = (gzip.open if opts.get('--bt').endswith('.gz')
					else open)(opts.get('--bt')).read().splitlines()
		stage.update(
				name='grammar',
				mode=mode,
				grammar=grammar,
				binarized='--bitpar' not in opts,
				backtransform=backtransform if len(args) < 4 else None,
				m=numparses,
				objective='mpd')
		if '--obj' in opts:
			stage.update(
					dop='reduction' if backtransform is None else 'doubledop',
					objective=opts['--obj'],
					m=int(opts.get('-m', 1)))
		stages.append(DictObj(stage))
		if backtransform:
			_ = stages[-1].grammar.getmapping(None,
				neverblockre=re.compile(b'.+}<'))
		parser = Parser(stages, verbosity=int(opts.get('--verbosity', 2)))
		morph = None
		del args[:2]
	else:
		from discodop.runexp import readparam
		directory = args[0]
		if not os.path.isdir(directory):
			raise ValueError('expected directory produced by "discodop runexp"')
		params = readparam(os.path.join(directory, 'params.prm'))
		params['resultdir'] = directory
		stages = params['stages']
		postagging = params['postagging']
		readgrammars(directory, stages, postagging,
				top=params.get('top', top))
		parser = Parser(stages,
				transformations=params.get('transformations'),
				binarization=params['binarization'],
				postagging=postagging if postagging and
					postagging.method == 'unknownword' else None,
				relationalrealizational=params.get('relationalrealizational'),
				verbosity=int(opts.get('--verbosity',
						params.get('verbosity', 2))))
		morph = params['morphology']
		del args[:1]
	infile = (io.open(args[0], encoding='utf-8')
			if len(args) >= 1 else sys.stdin)
	out = (io.open(args[1], 'w', encoding='utf-8')
			if len(args) == 2 else sys.stdout)
	doparsing(parser, infile, out, prob, oneline, tags, numparses,
			int(opts.get('--numproc', 1)), opts.get('--fmt', 'discbracket'),
			morph)

def doparsing(parser, infile, out, printprob, oneline, usetags, numparses,
		numproc, fmt, morphology):
	"""Parse sentences from file and write results to file, log to stdout."""
	times = []
	unparsed = 0
	if not oneline:
		infile = readinputbitparstyle(infile)
	infile = (line for line in infile if line.strip())
	if numproc == 1:
		initworker(parser, printprob, usetags, numparses, fmt, morphology)
		mymap = imap
	else:
		pool = multiprocessing.Pool(processes=numproc, initializer=initworker,
				initargs=(parser, printprob, usetags, numparses, fmt,
					morphology))
		mymap = pool.imap
	for output, noparse, sec, msg in mymap(worker, enumerate(infile)):
		if output:
			print(msg, file=sys.stderr)
			out.write(output)
			if noparse:
				unparsed += 1
			times.append(sec)
			sys.stderr.flush()
			out.flush()
	print('average time per sentence', sum(times) / len(times),
			'\nunparsed sentences:', unparsed,
			'\nfinished',
			file=sys.stderr)
	out.close()

def initworker(parser, printprob, usetags, numparses,
		fmt, morphology):
	"""Load parser for a worker process."""
	headrules = None
	if fmt in ('mst', 'conll'):
		headrules = readheadrules(parser.binarization.headrules)
	PARAMS.update(parser=parser, printprob=printprob,
			usetags=usetags, numparses=numparses, fmt=fmt,
			morphology=morphology, headrules=headrules)

def workerfunc(func):
	"""Wrap a multiprocessing worker function to produce a full traceback."""
	@wraps(func)
	def wrapper(*args, **kwds):
		"""Apply decorated function."""
		try:
			import faulthandler
			faulthandler.enable()  # Dump information on segfault.
		except (ImportError, io.UnsupportedOperation):
			pass
		# NB: only concurrent.futures on Python 3.3+ will exit gracefully.
		try:
			return func(*args, **kwds)
		except Exception:  # pylint: disable=W0703
			# Put traceback as string into an exception and raise that
			raise Exception('in worker process\n%s' %
					''.join(traceback.format_exception(*sys.exc_info())))
	return wrapper

@workerfunc
def worker(args):
	"""Parse a single sentence."""
	n, line = args
	line = line.strip()
	if not line:
		return '', True, 0, ''
	begin = time.clock()
	sent = line.split(' ')
	tags = None
	if PARAMS.usetags:
		sent, tags = zip(*(a.rsplit('/', 1) for a in sent))
	msg = 'parsing %d: %s' % (n, ' '.join(sent))
	result = list(PARAMS.parser.parse(sent, tags=tags))[-1]
	output = ''
	if result.noparse:
		msg += '\nNo parse for "%s"' % ' '.join(sent)
		if PARAMS.printprob:
			output += 'prob=%.16g\n' % result.prob
		output += '%s\t%s\n' % (result.parsetree, ' '.join(sent))
	else:
		output += ''.join(
				writetree(
					PARAMS.parser.postprocess(tree)[0], sent,
					n if PARAMS.numparses == 1 else ('%d-%d' % (n, k)),
					PARAMS.fmt, headrules=PARAMS.headrules,
					morphology=PARAMS.morphology,
					comment=('prob=%.16g' % prob) if PARAMS.printprob else None)
				for k, (tree, prob, _) in enumerate(nlargest(
					PARAMS.numparses, result.parsetrees, key=itemgetter(1))))
	sec = time.clock() - begin
	msg += '\n%g s' % sec
	return output, result.noparse, sec, msg

def readinputbitparstyle(infile):
	"""Yields lists of tokens, where '\\n\\n' identifies a sentence break.

	Lazy version of ``infile.read().split('\\n\\n')``."""
	sent = []
	for line in infile:
		line = line.strip()
		if not line:
			yield ' '.join(sent)
			sent = []
		sent.append(line)
	if sent:
		yield ' '.join(sent)

class Parser(object):
	"""A coarse-to-fine parser based on a given set of parameters.

	:param stages: a list of coarse-to-fine stages containing grammars and
		parameters.
	:param transformations: treebank transformations to reverse on parses.
	:param binarization: settings used for binarization; used for the
		tailmarker attribute which identifies heads in parser output.
	:param postagging: if given, an unknown word model is used to assign POS
		tags during parsing. The model consists of a DictObj with (at least)
		the following attributes:

		- unknownwordfun: function to produces signatures for unknown words.
		- lexicon: the set of known words in the grammar.
		- sigs: the set of word signatures occurring in the grammar.
	:param relationalrealizational: whether to reverse the RR-transform."""
	def __init__(self, stages, transformations=None, postagging=None,
			binarization=DictObj(tailmarker=None),
			relationalrealizational=None, verbosity=2):
		self.stages = stages
		self.transformations = transformations
		self.binarization = binarization
		self.postagging = postagging
		self.relationalrealizational = relationalrealizational
		self.verbosity = verbosity
		for stage in stages:
			if stage.mode.startswith('pcfg-bitpar'):
				exportbitpargrammar(stage)
			model = u'default'
			if stage.dop:
				if (stage.estimator == 'ewe'
						or stage.objective.startswith('sl-dop')):
					model = u'ewe'
				elif stage.estimator == 'bon':
					model = u'bon'
				if stage.objective == 'shortest':
					model = u'shortest'
			stage.grammar.switch(model, logprob=stage.mode != 'pcfg-posterior')
			if verbosity >= 3:
				logging.debug(stage.name)
				logging.debug(stage.grammar)

	def parse(self, sent, tags=None):
		"""Parse a sentence and perform postprocessing.

		Yields a dictionary from parse trees to probabilities for each stage.

		:param sent: a sequence of tokens.
		:param tags: if given, will be given to the parser instead of trying
			all possible tags."""
		if self.postagging:
			if self.transformations and 'FOLD-NUMBERS' in self.transformations:
				sent = ['000' if NUMBERRE.match(a) else a for a in sent]
			sent = replaceraretestwords(sent,
					self.postagging.unknownwordfun,
					self.postagging.lexicon, self.postagging.sigs)
		sent = list(sent)
		if tags is not None:
			tags = list(tags)
		chart = start = inside = outside = lastsuccessfulparse = None
		for n, stage in enumerate(self.stages):
			begin = time.clock()
			noparse = False
			parsetrees = fragments = None
			msg = '%s:\t' % stage.name.upper()
			model = u'default'
			if stage.dop:
				if (stage.estimator == 'ewe'
						or stage.objective.startswith('sl-dop')):
					model = u'ewe'
				elif stage.estimator == 'bon':
					model = u'bon'
				if stage.objective == 'shortest':
					model = u'shortest'
			x = stage.grammar.currentmodel
			stage.grammar.switch(model, logprob=stage.mode != 'pcfg-posterior')
			if stage.mode.startswith('pcfg-bitpar') and (
					not hasattr(stage, 'rulesfile')
					or x != stage.grammar.currentmodel):
				exportbitpargrammar(stage)
			if not stage.binarized and not stage.mode.startswith('pcfg-bitpar'):
				raise ValueError('non-binarized grammar requires use of bitpar')
			if not stage.prune or chart:
				if n != 0 and stage.prune and stage.mode != 'dop-rerank':
					beginprune = time.clock()
					if self.stages[n - 1].mode == 'pcfg-posterior':
						whitelist, msg1 = whitelistfromposteriors(
								inside, outside, start,
								self.stages[n - 1].grammar, stage.grammar,
								stage.k, stage.splitprune,
								self.stages[n - 1].markorigin,
								stage.mode.startswith('pcfg'))
					else:
						whitelist, msg1 = prunechart(
								chart, stage.grammar, stage.k,
								stage.splitprune,
								self.stages[n - 1].markorigin,
								stage.mode.startswith('pcfg'),
								self.stages[n - 1].mode == 'pcfg-bitpar-nbest')
					msg += '%s; %gs\n\t' % (msg1, time.clock() - beginprune)
				else:
					whitelist = None
				if stage.mode == 'pcfg':
					chart, msg1 = pcfg.parse(
							sent, stage.grammar, tags=tags,
							whitelist=whitelist if stage.prune else None)
				elif stage.mode == 'pcfg-posterior':
					inside, outside, start, msg1 = pcfg.doinsideoutside(
							sent, stage.grammar, tags=tags)
					chart = start
				elif stage.mode.startswith('pcfg-bitpar'):
					if stage.mode == 'pcfg-bitpar-forest':
						numderivs = 0
					elif (n == len(self.stages) - 1
							or not self.stages[n + 1].prune):
						numderivs = stage.m
					else:  # request 1000 nbest parses for CTF pruning
						numderivs = 1000
					chart, cputime, msg1 = pcfg.parse_bitpar(stage.grammar,
							stage.rulesfile.name, stage.lexiconfile.name,
							sent, numderivs,
							stage.grammar.start,
							stage.grammar.toid[stage.grammar.start], tags=tags)
					begin -= cputime
				elif stage.mode == 'plcfrs':
					chart, msg1 = plcfrs.parse(
							sent, stage.grammar, tags=tags,
							exhaustive=stage.dop or (
								n + 1 != len(self.stages)
								and self.stages[n + 1].prune),
							whitelist=whitelist,
							splitprune=stage.splitprune
								and self.stages[n - 1].split,
							markorigin=self.stages[n - 1].markorigin,
							estimates=(stage.estimates, stage.outside)
								if stage.estimates in ('SX', 'SXlrgaps')
								else None)
				elif stage.mode == 'dop-rerank':
					if chart:
						parsetrees = doprerank(chart, sent, stage.k,
								self.stages[n - 1].grammar, stage.grammar)
						msg1 = 're-ranked %d parse trees. ' % len(parsetrees)
				else:
					raise ValueError('unknown mode specified.')
				msg += '%s\n\t' % msg1
				if (n != 0 and not chart and not noparse
						and stage.split == self.stages[n - 1].split):
					logging.error('ERROR: expected successful parse. '
							'sent: %s\nstage: %s.', ' '.join(sent), stage.name)
					# raise ValueError('ERROR: expected successful parse. '
					# 		'sent %s, %s.' % (nsent, stage.name))
			if chart and stage.mode not in ('pcfg-posterior', 'dop-rerank'
					) and not (self.relationalrealizational and stage.split):
				begindisamb = time.clock()
				if stage.mode == 'pcfg-bitpar-nbest':
					if not stage.kbest or stage.sample:
						raise ValueError('sampling not possible with bitpar '
								'in nbest mode.')
					derivations = chart.rankededges[chart.root()]
					entries = [None] * len(derivations)
				else:
					derivations, entries = getderivations(chart, stage.m,
							kbest=stage.kbest, sample=stage.sample,
							derivstrings=stage.dop != 'doubledop'
									or self.verbosity >= 3
									or stage.objective == 'mcc')
				if self.verbosity >= 3:
					print('sent: %s\nstage: %s' % (' '.join(sent), stage.name))
					print('%d-best derivations:\n%s' % (
						min(stage.m, 100),
						'\n'.join('%d. %s %s' % (n + 1,
							('subtrees=%d' % abs(int(prob / log(0.5))))
							if stage.objective == 'shortest'
							else ('p=%g' % exp(-prob)), deriv)
						for n, (deriv, prob) in enumerate(derivations[:100]))))
					print('sum of probabitilies: %g\n' %
							sum(exp(-prob) for _, prob in derivations[:100]))
				if stage.objective == 'shortest':
					stage.grammar.switch(u'ewe' if stage.estimator == 'ewe'
							else u'default', True)
				parsetrees, msg1 = marginalize(
						stage.objective if stage.dop else 'mpd',
						derivations, entries, chart,
						sent=sent, tags=tags,
						backtransform=stage.backtransform,
						k=stage.m, sldop_n=stage.sldop_n,
						mcc_labda=stage.mcc_labda, mcc_labels=stage.mcc_labels,
						bitpar=stage.mode == 'pcfg-bitpar-nbest')
				msg += 'disambiguation: %s, %gs\n\t' % (
						msg1, time.clock() - begindisamb)
				if self.verbosity >= 3:
					besttrees = nlargest(100, parsetrees, key=itemgetter(1))
					print('100-best parse trees:\n%s' % '\n'.join(
							'%d. %s %s' % (n + 1, probstr(prob), treestr)
							for n, (treestr, prob, _) in enumerate(besttrees)))
					print('sum of probabitilies: %g\n' %
							sum((prob[1] if isinstance(prob, tuple) else prob)
								for _, prob, _ in besttrees))
			if self.verbosity >= 4:
				print('Chart:\n%s' % chart)
			if parsetrees:
				try:
					resultstr, prob, fragments = max(
							parsetrees, key=itemgetter(1))
					parsetree, noparse = self.postprocess(resultstr, n)
					if not all(a for a in parsetree.subtrees()):
						raise ValueError('empty nodes in tree: %s' % parsetree)
					if not len(parsetree.leaves()) == len(sent):
						raise ValueError('leaves missing. original tree: %s\n'
							'postprocessed: %r' % (resultstr, parsetree))
				except Exception:  # pylint: disable=W0703
					logging.error("something's amiss. %s", ''.join(
								traceback.format_exception(*sys.exc_info())))
					parsetree, prob, noparse = self.noparse(
							stage, sent, tags, lastsuccessfulparse)
				else:
					lastsuccessfulparse = parsetree
				msg += probstr(prob) + ' '
			else:
				fragments = None
				parsetree, prob, noparse = self.noparse(
						stage, sent, tags, lastsuccessfulparse)
			elapsedtime = time.clock() - begin
			msg += '%.2fs cpu time elapsed\n' % (elapsedtime)
			yield DictObj(name=stage.name, parsetree=parsetree, prob=prob,
					parsetrees=parsetrees, fragments=fragments,
					noparse=noparse, elapsedtime=elapsedtime, msg=msg)

	def postprocess(self, treestr, stage=-1):
		"""Take parse tree and apply postprocessing."""
		parsetree = Tree.parse(treestr, parse_leaf=int)
		if self.stages[stage].split:
			mergediscnodes(unbinarize(parsetree, childchar=':',
					expandunary=False))
		saveheads(parsetree, self.binarization.tailmarker)
		unbinarize(parsetree, expandunary=False)
		removefanoutmarkers(parsetree)
		if self.relationalrealizational:
			parsetree = rrbacktransform(parsetree,
					self.relationalrealizational['adjunctionlabel'])
		if self.transformations:
			reversetransform(parsetree, self.transformations)
		return parsetree, False

	def noparse(self, stage, sent, tags, lastsuccessfulparse):
		"""Return parse from previous stage or a dummy parse."""
		# use successful parse from earlier stage if available
		if lastsuccessfulparse is not None:
			parsetree = lastsuccessfulparse.copy(True)
		else:  # Produce a dummy parse for evaluation purposes.
			default = defaultparse([(n, t) for n, t
					in enumerate(tags or (len(sent) * ['NONE']))])
			parsetree = Tree.parse('(%s %s)' % (stage.grammar.start,
					default), parse_leaf=int)
		noparse = True
		prob = 1.0
		return parsetree, prob, noparse

def readgrammars(resultdir, stages, postagging=None, top='ROOT'):
	"""Read the grammars from a previous experiment.

	Expects a directory ``resultdir`` which contains the relevant grammars and
	the parameter file ``params.prm``, as produced by ``runexp``."""
	for n, stage in enumerate(stages):
		logging.info('reading: %s', stage.name)
		rules = gzip.open('%s/%s.rules.gz' % (resultdir, stage.name)).read()
		lexicon = codecs.getreader('utf-8')(gzip.open('%s/%s.lex.gz' % (
				resultdir, stage.name)))
		grammar = Grammar(rules, lexicon.read(),
				start=top, bitpar=stage.mode.startswith('pcfg')
				or re.match(r'[-.e0-9]+\b', rules), binarized=stage.binarized)
		backtransform = outside = None
		if stage.dop:
			if stage.estimates is not None:
				raise ValueError('not supported')
			if stage.dop == 'doubledop':
				backtransform = gzip.open('%s/%s.backtransform.gz' % (
						resultdir, stage.name)).read().splitlines()
				if n and stage.prune:
					_ = grammar.getmapping(stages[n - 1].grammar,
						striplabelre=re.compile(b'@.+$'),
						neverblockre=re.compile(b'^#[0-9]+|.+}<'),
						splitprune=stage.splitprune and stages[n - 1].split,
						markorigin=stages[n - 1].markorigin)
				else:
					# recoverfragments() relies on this mapping to identify
					# binarization nodes
					_ = grammar.getmapping(None,
						neverblockre=re.compile(b'.+}<'))
			elif n and stage.prune:  # dop reduction
				_ = grammar.getmapping(stages[n - 1].grammar,
					striplabelre=re.compile(b'@[-0-9]+$'),
					neverblockre=re.compile(stage.neverblockre)
						if stage.neverblockre else None,
					splitprune=stage.splitprune and stages[n - 1].split,
					markorigin=stages[n - 1].markorigin)
				if stage.mode == 'dop-rerank':
					grammar.getrulemapping(
							stages[n - 1].grammar, re.compile(br'@[-0-9]+\b'))
			probsfile = '%s/%s.probs.npz' % (resultdir, stage.name)
			if os.path.exists(probsfile):
				probmodels = np.load(probsfile)  # pylint: disable=no-member
				for name in probmodels.files:
					if name != 'default':
						grammar.register(unicode(name), probmodels[name])
		else:  # not stage.dop
			if n and stage.prune:
				_ = grammar.getmapping(stages[n - 1].grammar,
					neverblockre=re.compile(stage.neverblockre)
						if stage.neverblockre else None,
					splitprune=stage.splitprune and stages[n - 1].split,
					markorigin=stages[n - 1].markorigin)
			if stage.estimates in ('SX', 'SXlrgaps'):
				if stage.estimates == 'SX' and grammar.maxfanout != 1:
					raise ValueError('SX estimate requires PCFG.')
				if stage.mode != 'plcfrs':
					raise ValueError('estimates require parser w/agenda.')
				outside = np.load(  # pylint: disable=no-member
						'%s/%s.outside.npz' % (resultdir, stage.name))['outside']
				logging.info('loaded %s estimates', stage.estimates)
			elif stage.estimates:
				raise ValueError('unrecognized value; specify SX or SXlrgaps.')

		if stage.mode.startswith('pcfg-bitpar'):
			if grammar.maxfanout != 1:
				raise ValueError('bitpar requires a PCFG.')

		_sumsto1, msg = grammar.testgrammar()
		logging.info('%s: %s', stage.name, msg)
		stage.update(grammar=grammar, backtransform=backtransform,
				outside=outside)
	if postagging and postagging.method == 'unknownword':
		postagging.unknownwordfun = UNKNOWNWORDFUNC[postagging.model]
		postagging.lexicon = {w for w in stages[0].grammar.lexicalbyword
				if not w.startswith(UNK)}
		postagging.sigs = {w for w in stages[0].grammar.lexicalbyword
				if w.startswith(UNK)}

def exportbitpargrammar(stage):
	"""(re-)export bitpar grammar with current weights."""
	if not hasattr(stage, 'rulesfile'):
		stage.rulesfile = tempfile.NamedTemporaryFile()
		stage.lexiconfile = tempfile.NamedTemporaryFile()
	stage.rulesfile.seek(0)
	stage.rulesfile.truncate()
	if stage.grammar.currentmodel == 0:
		stage.rulesfile.write(stage.grammar.origrules)
	else:
		stage.rulesfile.writelines(
				'%g\t%s\n' % (weight, line.split(None, 1)[1])
				for weight, line in
				zip(stage.grammar.models[stage.grammar.currentmodel],
					stage.grammar.origrules.splitlines()))
	stage.rulesfile.flush()

	stage.lexiconfile.seek(0)
	stage.lexiconfile.truncate()
	lexicon = stage.grammar.origlexicon.replace(
			'(', '-LRB-').replace(')', '-RRB-')
	lexiconfile = codecs.getwriter('utf-8')(stage.lexiconfile)
	if stage.grammar.currentmodel == 0:
		lexiconfile.write(lexicon)
	else:
		weights = iter(stage.grammar.models[stage.grammar.currentmodel,
				stage.grammar.numrules:])
		lexiconfile.writelines('%s\t%s\n' % (line.split(None, 1)[0],
				'\t'.join('%s %g' % (tag, next(weights))
					for tag in line.split()[1::2]))
				for line in lexicon.splitlines())
	stage.lexiconfile.flush()

def probstr(prob):
	"""Render probability / number of subtrees as string."""
	if isinstance(prob, tuple):
		return 'subtrees=%d, p=%.4g ' % (abs(prob[0]), prob[1])
	return 'p=%.4g' % prob

def which(program):
	"""Return first match for program in search path."""
	for path in os.environ.get('PATH', os.defpath).split(":"):
		if path and os.path.exists(os.path.join(path, program)):
			return os.path.join(path, program)
	raise ValueError('%r not found in path; please install it.' % program)

__all__ = ['DictObj', 'Parser', 'doparsing', 'exportbitpargrammar',
		'initworker', 'probstr', 'readgrammars', 'readinputbitparstyle',
		'which', 'worker', 'workerfunc']

if __name__ == '__main__':
	main()


#!/usr/bin/env python
# coding: utf-8

# In[2]:

colorList = ["blue", "purple", "pink", "yellow", "green"]
for color in colorList:
    print(color)

# In[2]:

n = 0
numbers = list(range(10))
for num in numbers:
    print (num)

# In[1]:

n = 0
while n<10:
    print(n)
    n = n+1

# In[1]:

n = 0
while n <= 10:
    print(n)
    n = n + 1
else:
    print("greater than 10")

# In[3]:

n = 30
sum = 0
i = 1

while i <= n:
    sum = sum+i
    i=i+1

print(sum)

# In[ ]:

# In[ ]:


"""
Table of Contents

1. drop_null_columns: Drop columns that exceed a threshold of null values.
"""

from pyspark.sql import functions as F, DataFrame

from ..parsing.melting import melt

def drop_null_columns(df: DataFrame, threshold: float = 1.0, subset: list = None) -> DataFrame:
    """
    Drop columns that exceed a threshold of null values.

    Inputs
        df: DataFrame.
        threshold: Threshold value. If a column has at least this fraction of nulls, it will be removed.
        subset: List of columns to check. All others will be kept by default regardless of the null count.

    Output
        Updated DataFrame.

    Example
        df = df.drop_null_columns(df, 0.5, ["col1", "col2"])
        Will remove col1 or col2 if either is at least 50% null.
    """
    # If looking only at a subset of columns, downselect, otherwise, evaluate all columns.
    if subset is None:
        temp_df = df
    else:
        temp_df = df.select(subset)

    # List of columns being evaluated.
    columns_evaluated = temp_df.columns

    # Replace each column with a 1 if null, 0 otherwise.
    for col in columns_evaluated:
        temp_df = temp_df.withColumn(col, F.when(F.col(col).isNull(), 1).otherwise(0))

    # Sum the number of nulls, represented with a 1, for each column.
    null_count = temp_df.agg(*[F.sum(c).alias(c) for c in columns_evaluated])

    # Total row count.
    count = temp_df.count()

    # Pivot this null_count DataFrame to do a few operations to find when there are too many nulls.
    # The names of the columns will now be listed in a column called "categories".
    null_count = melt(null_count, value_vars=columns_evaluated)
    null_count = (
        null_count
        # Compute the fraction of rows that are null.
        .withColumn("fraction", F.col("values") / F.lit(count))
        # Keep those rows that meet or exceed the threshold.
        .where(F.col("fraction") >= F.lit(threshold))
        # Now, pivot again so that the column names are restored.
        .groupBy().pivot("categories").sum("fraction")
    )

    # Get the list of the columns that need to be dropped; drop and return.
    columns_to_drop = null_count.columns
    df = df.drop(*columns_to_drop)

    return df


# W.A.P in Python to count the total no. of words in a string. #

str = input("Enter the string :- ")
total = 1
for i in range(len(str)) :                      # len() function returns the length of the string.
    if(str[i]==" ") :
        total+=1
print("The total number of words in the string is ",total)

                 # ALTERNATIVE METHOD #

# str = input("Enter the string :- ")
# total = 1
# total = total + str.count(" ")                   # count() function is used to count the total number of occurances of any specified character.
# print("The total number of words in the string is ",total)


import numpy as np
from tqdm import tqdm

flip_inst = {}
flip_inst['e'] = [1, 0]
flip_inst['w'] = [-1, 0]
flip_inst['se'] = [0, -1]
flip_inst['sw'] = [-1, -1]
flip_inst['ne'] = [1, 1]
flip_inst['nw'] = [0, 1]

def flip_tile(instr, tiles):
    tile = np.array([0, 0])
    while instr:
        for fi, dir in flip_inst.items():
            if instr.startswith(fi):
                tile += dir
                instr = instr[len(fi):]
                continue

    if list(tile) in tiles:
        del tiles[tiles.index(list(tile))]
    else:
        tiles.append(list(tile))
    return tiles

def count_neighbors(matrix, x, y):
    num = 0
    for _, dir in flip_inst.items():
        if matrix[x + dir[0], y + dir[1]] == 1:
            num += 1
    return num

def evolve(matrix):
    to_flip = np.zeros(matrix.shape)
    for x in range(1, matrix.shape[0]-1):
        for y in range(1, matrix.shape[1]-1):
            neighbors = count_neighbors(matrix, x, y)
            # Any black tile with zero or more than 2 black tiles immediately
            # adjacent to it is flipped to white.
            if matrix[x, y] == 1:
                if neighbors == 0 or neighbors > 2:
                    to_flip[x, y] = 1
            # Any white tile with exactly 2 black tiles immediately adjacent to
            # it is flipped to black.
            else:
                if neighbors == 2:
                    to_flip[x, y] = 1
    matrix[to_flip == 1] = (matrix[to_flip == 1] + 1) % 2
    return matrix

if __name__ == "__main__":
    file_name = "test_24.txt"
    file_name = "input_24.txt"

    tiles = []
    for line in open(file_name):
        tiles = flip_tile(line.strip(), tiles)
    print(len(tiles))  # 521

    tiles = np.array(tiles)
    delx = max(tiles[:, 0]) - min(tiles[:, 0])
    dely = max(tiles[:, 1]) - min(tiles[:, 1])
    matrix = np.zeros([200 + delx, 200 + dely], np.int)
    for t in tiles:
        matrix[t[0]+100, t[1]+100] = 1

    for _ in tqdm(range(100)):
        matrix = evolve(matrix)
    print(np.sum(matrix))  # 4242


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from blog import apiviews

router = DefaultRouter()
router.register('posts', apiviews.PostViewSet)
router.register('comments', apiviews.CommentViewSet)
router.register('replies', apiviews.ReplyViewSet)
router.register('users', apiviews.UserViewSet )

urlpatterns = [
    path('', include(router.urls)),
    path('rest-auth/', include('rest_auth.urls')),
    path('rest-auth/registration/', include('rest_auth.registration.urls')),
    path('categories', apiviews.CategoryListView.as_view(), name='categories-list'),
    path('category-filter/<int:pk>/', apiviews.PostsInCategoryView.as_view(), name='category-detail')
]

# urlpatterns += router.urls


import glob
import numpy as np
import pandas as pd
import nibabel as nib

import torch
from torch.utils.data import Dataset

# dataset class for the GenericObjectDecoding dataset
class GODData(Dataset):
    FEATURES_PATH = "data/ds001246/derivatives/preproc-spm/output"
    TARGETS_PATH = "data/ds001246"
    TRAIN_CATEGORIES_PATH = "data/ds001246/stimulus_ImageNetTraining.csv"
    TEST_CATEGORIES_PATH = "data/ds001246/stimulus_ImageNetTest.csv"

    def __init__(
            self,
            subject="01",
            session_id="01",
            task="perception",
            train=True,
            limit_size=None,
        ):
        session = f"{task}{'Training' if train else 'Test'}{session_id}"

        # load data
        feature_runs = sorted(glob.glob(f"{self.FEATURES_PATH}/sub-{subject}/ses-{session}/func/*"))
        target_runs = sorted(glob.glob(f"{self.TARGETS_PATH}/sub-{subject}/ses-{session}/func/*events*"))
        categories = pd.read_csv(self.TRAIN_CATEGORIES_PATH if train else self.TEST_CATEGORIES_PATH, sep="\t", header=None)

        # process features and targets
        features = []
        targets = []

        for f_run, t_run in zip(feature_runs, target_runs):
            features_run = nib.load(f_run).get_fdata()
            targets_run = pd.read_csv(t_run, sep="\t")

            # remove resting states
            features_run_pp = features_run[:, :, :, 8:-2]
            targets_run_pp = targets_run[targets_run["event_type"] != "rest"]

            # reshape features into (N, C, D, W, H)
            features_run_pp = features_run_pp.transpose(3, 2, 1, 0).reshape(-1, 3, 50, 64, 64)

            # extract category labels
            targets_run_pp = targets_run_pp.merge(categories, left_on="stim_id", right_on=1)[2]
            targets_run_pp = targets_run_pp.to_numpy().reshape(-1, 1)

            features.append(features_run_pp)
            targets.append(targets_run_pp)

        features = np.vstack(features)
        targets = np.vstack(targets)

        # convert and store as tensors
        self.features = torch.from_numpy(features).float()
        self.targets = torch.from_numpy(targets).long() - 1

        # flatten targets
        self.targets = self.targets.squeeze()

        # limit dataset size
        if limit_size is not None:
            self.features = self.features[:limit_size]
            self.targets = self.targets[:limit_size]

    def __len__(self):
        return len(self.features)

    def __getitem__(self, index):
        feature = self.features[index]
        target = self.targets[index]
        return feature, target


"""
Difficulty: Easy

Given a signed 32-bit integer x, return x with its digits reversed.
If reversing x causes the value to go outside the signed 32-bit integer range [-231, 231 - 1], then return 0.

Assume the environment does not allow you to store 64-bit integers (signed or unsigned).

Example 1:
Input: x = 123
Output: 321

Example 2:
Input: x = -123
Output: -321

Example 3:
Input: x = 120
Output: 21

Constraints:
    * -2^31 <= x <= 2^31 - 1
"""

class Solution:

    # approach 1: divide by 10, get remainder, add to result
    def reverse(self, x: int) -> int:
        result = ""
        y = abs(x)
        while y > 0:
            quot, rem = y // 10, y % 10
            result += str(rem)
            y = quot

        if not result:
            return 0

        result = int(result)
        if result < (-2**31) or result > (2**31 - 1):
            return 0

        if x < 0:
            result = -1 * result
        return result

    # approach 2: convert to string, reverse, convert back to int
    def reverse(self, x: int) -> int:
        if x < 0:
            return -1 * int(str(x)[1:][::-1])
        else:
            return int(str(x)[::-1])


import os
import numpy as np
import pandas as pd
from PIL import Image
from collections import Counter

import torch
from torch.utils.data import Dataset
from torch.nn.utils.rnn import pad_sequence
import torchvision.transforms as T

import spacy

spacy_eng = spacy.load("en_core_web_sm")

# defining the transform to be applied
transforms = T.Compose([
    T.Resize(226),
    T.RandomCrop(224),
    T.ToTensor(),
    T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])

transforms_advanced = T.Compose([
    T.Resize(226),
    T.RandomCrop(224),
    T.ToTensor(),
    T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    T.RandomAffine(10),
    T.RandomGrayscale(0.05),
    T.RandomHorizontalFlip(0.05),
    T.RandomVerticalFlip(0.05),
    T.GaussianBlur(5),
    T.RandomErasing(0.05)
])

class Vocabulary:
    def __init__(self, freq_threshold):
        # setting the pre-reserved tokens int to string tokens
        self.pad_idx = 0
        self.sos_idx = 1
        self.eos_idx = 2
        self.unk_idx = 3
        self.pad = "<PAD>"
        self.sos = "<SOS>"
        self.eos = "<EOS>"
        self.unk = "<UNK>"
        self.itos = {self.pad_idx: self.pad,
                     self.sos_idx: self.sos,
                     self.eos_idx: self.eos,
                     self.unk_idx: self.unk}

        # string to int tokens
        # its reverse dict self.itos
        self.stoi = {v: k for k, v in self.itos.items()}

        self.freq_threshold = freq_threshold

    def __len__(self):
        return len(self.itos)

    def size(self):
        return len(self.itos)

    @staticmethod
    def tokenize(text):
        return [token.text.lower() for token in spacy_eng.tokenizer(text)]

    def build_vocab(self, sentence_list):
        frequencies = Counter()
        idx = 4

        for sentence in sentence_list:
            for word in self.tokenize(sentence):
                frequencies[word] += 1

                # add the word to the vocab if it reaches minum frequecy threshold
                if frequencies[word] == self.freq_threshold:
                    self.stoi[word] = idx
                    self.itos[idx] = word
                    idx += 1

    def numericalize(self, text):
        """ For each word in the text corresponding index token for that word form the vocab built as list """
        tokenized_text = self.tokenize(text)
        return [self.stoi[token] if token in self.stoi else self.stoi["<UNK>"] for token in tokenized_text]

class FlickrDataset(Dataset):
    """
    FlickrDataset
    """

    def __init__(self,
                 root_dir,
                 captions_file,
                 transform=None,
                 freq_threshold=5,
                 vocab=None,
                 data_limit=None,
                 do_augmentation=False,
                 augmentation_probability=0.2):
        self.root_dir = root_dir
        self.df = pd.read_csv(captions_file)
        self.transform = transform
        self.random = np.random.RandomState()
        self.do_augmentation = do_augmentation
        self.augmentation_probability = augmentation_probability

        # Get image and caption colum from the dataframe
        self.imgs = self.df["image"]
        self.captions = self.df["caption"]
        # If needed truncating the data for faster running
        if data_limit is not None:
            self.imgs = self.imgs[:data_limit]
            self.captions = self.captions[:data_limit]

        # Initialize vocabulary and build vocab
        if vocab is None:
            self.vocab = Vocabulary(freq_threshold)
            self.vocab.build_vocab(self.captions.tolist())
        else:
            self.vocab = vocab

    def __len__(self):
        # return len(self.df)
        return self.imgs.shape[0]

    def __getitem__(self, idx):
        caption = self.captions[idx]
        img_name = self.imgs[idx]
        img_location = os.path.join(self.root_dir, img_name)
        img_pil = Image.open(img_location).convert("RGB")

        # do some random augmentations
        if not self.do_augmentation:
            img = transforms(img_pil)
        else:
            img = transforms_advanced(img_pil)

        # numericalize the caption text
        caption_vec = []
        caption_vec += [self.vocab.stoi["<SOS>"]]
        caption_vec += self.vocab.numericalize(caption)
        caption_vec += [self.vocab.stoi["<EOS>"]]

        return img, torch.tensor(caption_vec)

class CapsCollate:
    """
    Collate to apply the padding to the captions with dataloader
    """

    def __init__(self, vocab, batch_first=False, max_len=0):
        self.pad_idx = vocab.pad_idx
        self.eos_idx = vocab.eos_idx
        self.batch_first = batch_first
        self._max_len = max_len

    def __call__(self, batch):
        imgs = [item[0].unsqueeze(0) for item in batch]
        imgs = torch.cat(imgs, dim=0)

        targets = [item[1] for item in batch]
        targets = pad_sequence(targets, batch_first=self.batch_first, padding_value=self.pad_idx)
        if self._max_len > 0:
            if targets.shape[1] >= self._max_len:
                targets = targets[:, :self._max_len]
            else:
                pad_tensor = torch.ones(size=(targets.shape[0], self._max_len - targets.shape[1]),
                                        dtype=torch.long) * self.pad_idx
                targets = torch.cat([targets, pad_tensor], dim=1)

        targets[:, -1] = targets[:, -1].where(targets[:, -1] == self.pad_idx, torch.tensor(self.eos_idx))
        return imgs, targets


import os
import pickle
import argparse
import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from Model.INPLIM import Doctor
from data_utils import CodeLevelDataset
from utils import train_eval

def args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_root', type=str, default='',
                        help='Set the root path of the dataset')
    parser.add_argument('--devices', type=str, default='cpu',
                        help='Setting the IDs of GPU devices.')
    parser.add_argument('--epochs', type=int, default=15,
                        help='Setting the number of epochs to run.')
    parser.add_argument('--batch_size', type=int, default=512,
                        help='Setting the mini-batch size.')
    parser.add_argument('--weight_decay', type=float, default=1e-3,
                        help='Setting weight decay')
    parser.add_argument('--lr', type=float, default=1e-3,
                        help='Setting the learning rate.')
    parser.add_argument('--dim', type=int, default=128,
                        help='Setting the inner dim of the model.')
    parser.add_argument('--max_len', type=int, default=200,
                        help='Setting the maximum number of code to use for a patient.')
    parser.add_argument('--drop_context', type=float, default=0.3,
                        help='Setting drop rate of the context-aware branch.')
    parser.add_argument('--drop_time', type=float, default=0.3,
                        help='Setting drop rate of the time-aware branch.')
    parser.add_argument('--save_model', action='store_true',
                        help='Whether to save the parameters of the trained model.',
                        default=True)
    parser.add_argument('--save_dir', type=str, default='./saved_models',
                        help='Setting the dir of saving trained model.')
    return parser

def main(opts):
    if opts.devices != 'cpu':
        os.environ['CUDA_VISIBLE_DEVICES'] = opts.devices

    dataset = pickle.load(open(opts.data_root, 'rb'))
    train_set = CodeLevelDataset(dataset=dataset, max_len=opts.max_len, phase='train')
    valid_set = CodeLevelDataset(dataset=dataset, max_len=opts.max_len, phase='val')
    test_set = CodeLevelDataset(dataset=dataset, max_len=opts.max_len, phase='test')

    train_loader = DataLoader(train_set, batch_size=opts.batch_size, num_workers=2, shuffle=True)
    val_loader = DataLoader(valid_set, batch_size=1, num_workers=1, shuffle=False)
    test_loader = DataLoader(test_set, batch_size=1, num_workers=1, shuffle=False)

    net = Doctor(features=dataset['features'], out_dim=2, emb_dim=opts.dim, dropout_context=opts.drop_context,
                 dropout_time=opts.drop_time)
    criterion = torch.nn.CrossEntropyLoss()
    optimizer = optim.Adam(net.parameters(), lr=opts.lr, weight_decay=opts.weight_decay, eps=0)

    if opts.devices != 'cpu':
        net = torch.nn.DataParallel(net).cuda()
        criterion = criterion.cuda()

    best_auc, best_epoch, best_test_nll, best_test_auc_roc = 0, 0, 0, 0
    model_dict = {}
    for epoch in range(opts.epochs):
        train_eval(opts, net, criterion, optimizer, train_loader, epoch, phase='Train')
        _, auc = train_eval(opts, net, criterion, optimizer, val_loader, epoch, phase='Valid')

        if auc > best_auc:
            best_auc, best_epoch = auc, epoch
            best_test_nll, best_test_auc_roc = train_eval(opts, net, criterion, optimizer, test_loader, epoch,
                                                          phase='Test')
            model_dict['opts'] = opts
            model_dict['states'] = net.state_dict()
    print('Best Test NLL:{:.4f}\t Best AUROC:{:.4f}'.format(best_test_nll, best_test_auc_roc))

    if not os.path.exists(opts.save_dir):
        os.makedirs(opts.save_dir)

    if opts.save_model:
        torch.save(model_dict, os.path.join(opts.save_dir, 'Model-AUC-{:.4f}.pth'.format(best_test_auc_roc)))

if __name__ == '__main__':
    opts = args().parse_args()
    main(opts)


# -*- coding: utf-8 -*-
"""
Created on Thu Oct 14 14:19:49 2021

@author: 姜高晓
"""
import numpy as np
from scipy.fftpack import fft,ifft
from matplotlib import pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.model_selection import ParameterGrid, GridSearchCV
R=np.array([ [0,0,1300],
             [5000,0,1700],
             [0,5000,1700],
             [5000,5000,1300]
             ])
#%%  梯度下降法
def L(r,A):
    s=0
    for i in range(4):
        s+=(np.sqrt(((r-R[i])**2).sum())-A[i])**2
    return s
def dL(r,A):
    s=np.zeros(3)
    for i in range(4):##i表示标号
        s[0]+=1*(np.sqrt(((r-R[i])**2).sum())-A[i])*(r[0]-R[i][0])/(np.sqrt(((r-R[i])**2).sum()))*1
        s[1]+=1*(np.sqrt(((r-R[i])**2).sum())-A[i])*(r[1]-R[i][1])/(np.sqrt(((r-R[i])**2).sum()))*1
        s[2]+=1*(np.sqrt(((r-R[i])**2).sum())-A[i])*(r[2]-R[i][2])/(np.sqrt(((r-R[i])**2).sum()))*1
    return s
alpha=0.1
Nstep=800
pre=[]
for i in range(1,325):
    d=np.loadtxt("./out正常/"+str(i//1)+".正常.txt")
    #d=np.loadtxt("./out异常/"+str(i//1)+".异常.txt")
    d1=np.zeros(4)
    d1[0]=d[:,1].mean()
    d1[1]=d[:,2].mean()
    d1[2]=d[:,3].mean()
    d1[3]=d[:,4].mean()
    rn=[np.array([2000,2000,2000])]
    for i in range(Nstep):
        rn.append(rn[-1]-alpha*dL(rn[-1],d1))
    rn=np.array(rn)
    l=[rn[-50:,0].mean(),rn[-50:,1].mean(),rn[-50:,2].mean()]
    lstd=np.array([rn[-500:,0].var(),rn[-500:,1].var(),rn[-500:,2].var()])
    pre.append([L(l,d1),lstd.sum()])
pre=np.array(pre)
kam1=pre[:,0].min()
kam2=pre[:,1].max()
#mean=
pre[:,0]=pre[:,0]/kam1
pre[:,1]=pre[:,1]/kam2

pre1=np.array(pre)
a1=pre[:,0].mean()
a2=pre[:,1].mean()
#%%朴素值
pre=[]
for i in range(1,325):
    #d=np.loadtxt("./out正常/"+str(i//1)+".正常.txt")
    d=np.loadtxt("./out异常/"+str(i//1)+".异常.txt")
    d1=np.zeros(4)
    d1[0]=d[:,1].mean()
    d1[1]=d[:,2].mean()
    d1[2]=d[:,3].mean()
    d1[3]=d[:,4].mean()
    rn=[np.array([2000,2000,2000])]
    for i in range(Nstep):
        rn.append(rn[-1]-alpha*dL(rn[-1],d1))
    rn=np.array(rn)
    l=[rn[-50:,0].mean(),rn[-50:,1].mean(),rn[-50:,2].mean()]
    lstd=np.array([rn[-500:,0].var(),rn[-500:,1].var(),rn[-500:,2].var()])
    pre.append([L(l,d1),lstd.sum()])
pre=np.array(pre)
pre[:,0]=pre[:,0]/kam1
pre[:,1]=pre[:,1]/kam2
#plt.plot(pre)
#plt.show()
b1=pre[:,0].mean()
b2=pre[:,1].mean()
# 126 124 53
# 118 128 63
# 119 122 138  正常
#1.4 1.4 0.57

pre2=np.array(pre)

a=open("./4.txt",encoding='utf-8').read().splitlines()
for i in range(len(a)):
    a[i]=a[i].split(':')
a=np.array(a)
b=np.zeros((len(a[:,0])//4,5))
for i in range(len(a[:,0])//4):
    b[i,0]=a[4*i,1]
    b[i,1]=a[4*i,5]
    b[i,2]=a[4*i+1,5]
    b[i,3]=a[4*i+2,5]
    b[i,4]=a[4*i+3,5]
pre=[]
for i in range(10):
    rn=[np.array([2000,2000,2000])]
    d1=b[i,1:]
    for j in range(Nstep):
        rn.append(rn[-1]-alpha*dL(rn[-1],d1))
    rn=np.array(rn)
    l=[rn[-50:,0].mean(),rn[-50:,1].mean(),rn[-50:,2].mean()]
    lstd=np.array([rn[-500:,0].var(),rn[-500:,1].var(),rn[-500:,2].var()])
    pre.append([L(l,d1),lstd.sum()])
pre=np.array(pre)
pre[:,0]=pre[:,0]/kam1
pre[:,1]=pre[:,1]/kam2
out=pre

aaaaa=out[:,0]*out[:,1]
label=[]
for i in range(10):
    A1=len(np.where(pre1[:,0]>out[i,0])[0])
    A2=(len(np.where(pre2[:,0]>out[i,0])[0]))
    label.append(A2/(A1+A2))

np.array(label)>0.5

math/4.2╖╓└α╞≈╫╘╢¿╥σ╖╓└α╞≈.py

4.2╖╓└α╞≈╫╘╢¿╥σ╖╓└α╞≈.py


# This is mostly lifted from django-storages' sftp backend: Their license:
#
# SFTP storage backend for Django.
# Author: Brent Tubbs <brent.tubbs@gmail.com>
# License: MIT
#
# Modeled on the FTP storage by Rafal Jonca <jonca.rafal@gmail.com>
from __future__ import print_function
try:
    import ssh
except ImportError:
    import paramiko as ssh

import os
import posixpath
import warnings

from django.conf import settings
from django.core.files.base import File
try:
    from io import StringIO
except ImportError:
    # Python 2 fallbacks
    from cStringIO import StringIO

from localdevstorage.base import BaseStorage

class SftpStorage(BaseStorage):
    def __init__(self, location=None, base_url=None, user=None, host=None, root_path=None):
        warnings.warn(
            'The SFTP backend is unsupported and untested. '
            'Usage is not recommended!'
        )
        self._host = host or settings.LOCALDEVSTORAGE_SFTP_HOST
        self._root_path = root_path or settings.LOCALDEVSTORAGE_SFTP_ROOT_PATH

        # if present, settings.SFTP_STORAGE_PARAMS should be a dict with params
        # matching the keyword arguments to paramiko.SSHClient().connect().  So
        # you can put username/password there.  Or you can omit all that if
        # you're using keys.
        self._params = getattr(settings, 'SFTP_STORAGE_PARAMS', {})
        self._params['username'] = user or settings.LOCALDEVSTORAGE_SFTP_USER

        # for now it's all posix paths.  Maybe someday we'll support figuring
        # out if the remote host is windows.
        self._pathmod = posixpath
        super(SftpStorage, self).__init__(location, base_url)

    def _connect(self):
        self._ssh = ssh.SSHClient()

        # automatically add host keys from current user.
        self._ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))

        # and automatically add new host keys for hosts we haven't seen before.
        self._ssh.set_missing_host_key_policy(ssh.AutoAddPolicy())

        try:
            self._ssh.connect(self._host, **self._params)
        except ssh.AuthenticationException as e:
            raise
        except Exception as e:
            print(e)

        if not hasattr(self, '_sftp'):
            self._sftp = self._ssh.open_sftp()

    @property
    def sftp(self):
        """Lazy SFTP connection"""
        if not hasattr(self, '_sftp'):
            self._connect()
        return self._sftp

    def _get(self, name):
        try:
            return SFTPStorageFile(name, self, 'rb')
        except IOError:
            pass

    def _exists_upstream(self, name):
        try:
            f = SFTPStorageFile(name, self, 'rb')
            f.close()
            return True
        except Exception:
            return False

    def _read(self, name):
        remote_path = self._remote_path(name)
        return self.sftp.open(remote_path, 'rb')

    def _remote_path(self, name):
        return self._join(self._root_path, name)

    def _join(self, *args):
        # Use the path module for the remote host type to join a path together
        return self._pathmod.join(*args)

class SFTPStorageFile(File):
    def __init__(self, name, storage, mode):
        self._name = name
        self._storage = storage
        self._mode = mode
        self._is_dirty = False
        self.file = StringIO()
        self._is_read = False

    @property
    def size(self):
        if not hasattr(self, '_size'):
            self._size = self._storage.size(self._name)
        return self._size

    def read(self, num_bytes=None):
        if not self._is_read:
            self.file = self._storage._read(self._name)
            self._is_read = True

        return self.file.read(num_bytes)

    def write(self, content):
        raise NotImplementedError

    def close(self):
        if self._is_dirty:
            self._storage._save(self._name, self.file.getvalue())
        self.file.close()


import cv2
import numpy as np

img = cv2.imread('images/saitama.jpg')
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)  # convert it to hsv

width = img.shape[1]
height = img.shape[0]
channel = img.shape[2]

increase_v = 40
decrease_s = 10
step = 2

# bien doi hinh anh
print("chon huong di cua animation: ")
print("1.Left -> Right")
print("2.Right -> Left")
print("3.Down")
print("4.Up")
flag = input()

if flag == '3':
    # huong di xuong
    for y in range(1, height, step):
        h, s, v = cv2.split(hsv[0: y, :])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[0: y, :] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
elif flag == '1':
    # huong sang phai
    for x in range(1, width, step):
        h, s, v = cv2.split(hsv[:, 0: x])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[:, 0:x] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
elif flag == '2':
    # huong sang trai
    for x in range(width - 2, 0, -step):
        h, s, v = cv2.split(hsv[:, x: -1])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[:, x:-1] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
elif flag == '4':
    # huong len tren
    for y in range(height - 2, 0, -step):
        h, s, v = cv2.split(hsv[y: -1, :])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[y: -1, :] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
else:
    print('nhap cac gia tri tu 1 den 4')
cv2.destroyAllWindows()


# 해시
import sys
input = sys.stdin.readline
n,m = map(int, input().split())
graph = {}
for _ in range(n):
    address, num = input().rstrip().split()
    graph[address] = num

for _ in range(m):
    temp = input().rstrip()
    print(graph[temp])

백준 문제/해시/비밀번호 찾기.py


import unittest

import bioschemas
from bioschemas.extractors import ExtractorFromHtml

config = bioschemas.DEFAULT_CONFIG

class TestExtractors(unittest.TestCase):
    def test_jsonld_extraction_from_html(self):
        html = '''<script type="application/ld+json">
        {
          "@context": "http://bioschemas.org",
          "@type": "PhysicalEntity",
          "name": "Gene arcA E. coli str. K-12 substr. MG1655 b4401",
          "additionalType": "http://www.ebi.ac.uk/ols/ontologies/so/terms?obo_id=SO:0000704",
          "identifier": "b4401",
          "url": "http://localhost:8080/synbiomine/report.do?id=2026346"
        }
        </script>
        '''

        e = ExtractorFromHtml(config)
        jsonlds = e._extract_jsonld_from_html(html)
        self.assertEqual(len(jsonlds), 1)

        jsonld = jsonlds[0]
        self.assertEqual(jsonld['name'], 'Gene arcA E. coli str. K-12 substr. MG1655 b4401')
        self.assertEqual(jsonld['additionalType'], 'http://www.ebi.ac.uk/ols/ontologies/so/terms?obo_id=SO:0000704')
        self.assertEqual(jsonld['identifier'], 'b4401')
        self.assertEqual(jsonld['url'], 'http://localhost:8080/synbiomine/report.do?id=2026346')


whiskyPriceInBgn = float(input())
beerLiters = float(input())
wineLiters = float(input())
rakiaLiters = float(input())
whiskyLiters = float(input())

rakiaPrice = whiskyPriceInBgn / 2
winePrice = rakiaPrice - (0.4 * rakiaPrice)
beerPrice = rakiaPrice - (0.8 * rakiaPrice)

totalSum = (whiskyPriceInBgn * whiskyLiters) + (beerLiters * beerPrice) + (wineLiters * winePrice) + (rakiaPrice * rakiaLiters)
print(f"{totalSum:.2f}")

Python Basics October 2018/Python-Simple-Operations-and-Calculations-Exercise/AlcoholMarket.py