from typing import Optional, Tuple, Union

import torch.nn as nn
from diffusers.models import UNet2DConditionModel
from diffusers.models.unet_2d_blocks import UNetMidBlock2DCrossAttn
from diffusers.models.embeddings import Timesteps, TimestepEmbedding
from diffusers.configuration_utils import register_to_config

from blocks import get_down_block, get_up_block

class VideoLDM(UNet2DConditionModel):
    @register_to_config
    def __init__(
        self,
        sample_size: Optional[int] = None,
        in_channels: int = 4,
        out_channels: int = 4,
        center_input_sample: bool = False,
        flip_sin_to_cos: bool = True,
        freq_shift: int = 0,
        down_block_types: Tuple[str] = (
            "CrossAttnDownBlock2D", # -> VideoLDMDownBlock
            "CrossAttnDownBlock2D", # -> VideoLDMDownBlock
            "CrossAttnDownBlock2D", # -> VideoLDMDownBlock
            "DownBlock2D",
        ),
        mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn",
        up_block_types: Tuple[str] = (
            "UpBlock2D",
            "CrossAttnUpBlock2D", # -> VideoLDMUpBlock
            "CrossAttnUpBlock2D", # -> VideoLDMUpBlock
            "CrossAttnUpBlock2D", # -> VideoLDMUpBlock
        ),
        only_cross_attention: Union[bool, Tuple[bool]] = False,
        block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
        layers_per_block: Union[int, Tuple[int]] = 2,
        downsample_padding: int = 1,
        mid_block_scale_factor: float = 1,
        act_fn: str = "silu",
        norm_num_groups: Optional[int] = 32,
        norm_eps: float = 1e-5,
        cross_attention_dim: Union[int, Tuple[int]] = 1280,
        encoder_hid_dim: Optional[int] = None,
        attention_head_dim: Union[int, Tuple[int]] = 8,
        dual_cross_attention: bool = False,
        use_linear_projection: bool = False,
        class_embed_type: Optional[str] = None,
        addition_embed_type: Optional[str] = None,
        num_class_embeds: Optional[int] = None,
        upcast_attention: bool = False,
        resnet_time_scale_shift: str = "default",
        resnet_skip_time_act: bool = False,
        resnet_out_scale_factor: int = 1.0,
        time_embedding_type: str = "positional",
        time_embedding_dim: Optional[int] = None,
        time_embedding_act_fn: Optional[str] = None,
        timestep_post_act: Optional[str] = None,
        time_cond_proj_dim: Optional[int] = None,
        conv_in_kernel: int = 3,
        conv_out_kernel: int = 3,
        projection_class_embeddings_input_dim: Optional[int] = None,
        class_embeddings_concat: bool = False,
        mid_block_only_cross_attention: Optional[bool] = None,
        cross_attention_norm: Optional[str] = None,
        addition_embed_type_num_heads=64,
    ):
        super().__init__()

        self.sample_size = sample_size

        # Check inputs
        if len(down_block_types) != len(up_block_types):
            raise ValueError(
                f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}."
            )

        if len(block_out_channels) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."
            )

        if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."
            )

        if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}."
            )

        if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}."
            )

        if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types):
            raise ValueError(
                f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}."
            )

        # input
        conv_in_padding = (conv_in_kernel - 1) // 2
        self.conv_in = nn.Conv2d(
            in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding
        )

        # time
        if time_embedding_type == "fourier":
            time_embed_dim = time_embedding_dim or block_out_channels[0] * 2
            if time_embed_dim % 2 != 0:
                raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.")
            self.time_proj = GaussianFourierProjection(
                time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos
            )
            timestep_input_dim = time_embed_dim
        elif time_embedding_type == "positional":
            time_embed_dim = time_embedding_dim or block_out_channels[0] * 4

            self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
            timestep_input_dim = block_out_channels[0]
        else:
            raise ValueError(
                f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`."
            )

        self.time_embedding = TimestepEmbedding(
            timestep_input_dim,
            time_embed_dim,
            act_fn=act_fn,
            post_act_fn=timestep_post_act,
            cond_proj_dim=time_cond_proj_dim,
        )

        if encoder_hid_dim is not None:
            self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)
        else:
            self.encoder_hid_proj = None

        # class embedding
        if class_embed_type is None and num_class_embeds is not None:
            self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
        elif class_embed_type == "timestep":
            self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn)
        elif class_embed_type == "identity":
            self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
        elif class_embed_type == "projection":
            if projection_class_embeddings_input_dim is None:
                raise ValueError(
                    "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"
                )
            # The projection `class_embed_type` is the same as the timestep `class_embed_type` except
            # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings
            # 2. it projects from an arbitrary input dimension.
            #
            # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.
            # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.
            # As a result, `TimestepEmbedding` can be passed arbitrary vectors.
            self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
        elif class_embed_type == "simple_projection":
            if projection_class_embeddings_input_dim is None:
                raise ValueError(
                    "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set"
                )
            self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim)
        else:
            self.class_embedding = None

        if addition_embed_type == "text":
            if encoder_hid_dim is not None:
                text_time_embedding_from_dim = encoder_hid_dim
            else:
                text_time_embedding_from_dim = cross_attention_dim

            self.add_embedding = TextTimeEmbedding(
                text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads
            )
        elif addition_embed_type is not None:
            raise ValueError(f"addition_embed_type: {addition_embed_type} must be None or 'text'.")

        if time_embedding_act_fn is None:
            self.time_embed_act = None
        elif time_embedding_act_fn == "swish":
            self.time_embed_act = lambda x: F.silu(x)
        elif time_embedding_act_fn == "mish":
            self.time_embed_act = nn.Mish()
        elif time_embedding_act_fn == "silu":
            self.time_embed_act = nn.SiLU()
        elif time_embedding_act_fn == "gelu":
            self.time_embed_act = nn.GELU()
        else:
            raise ValueError(f"Unsupported activation function: {time_embedding_act_fn}")

        self.down_blocks = nn.ModuleList([])
        self.up_blocks = nn.ModuleList([])

        if isinstance(only_cross_attention, bool):
            if mid_block_only_cross_attention is None:
                mid_block_only_cross_attention = only_cross_attention

            only_cross_attention = [only_cross_attention] * len(down_block_types)

        if mid_block_only_cross_attention is None:
            mid_block_only_cross_attention = False

        if isinstance(attention_head_dim, int):
            attention_head_dim = (attention_head_dim,) * len(down_block_types)

        if isinstance(cross_attention_dim, int):
            cross_attention_dim = (cross_attention_dim,) * len(down_block_types)

        if isinstance(layers_per_block, int):
            layers_per_block = [layers_per_block] * len(down_block_types)

        if class_embeddings_concat:
            # The time embeddings are concatenated with the class embeddings. The dimension of the
            # time embeddings passed to the down, middle, and up blocks is twice the dimension of the
            # regular time embeddings
            blocks_time_embed_dim = time_embed_dim * 2
        else:
            blocks_time_embed_dim = time_embed_dim

        # down
        output_channel = block_out_channels[0]
        for i, down_block_type in enumerate(down_block_types):
            input_channel = output_channel
            output_channel = block_out_channels[i]
            is_final_block = i == len(block_out_channels) - 1

            down_block = get_down_block(
                down_block_type,
                num_layers=layers_per_block[i],
                in_channels=input_channel,
                out_channels=output_channel,
                temb_channels=blocks_time_embed_dim,
                add_downsample=not is_final_block,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                resnet_groups=norm_num_groups,
                cross_attention_dim=cross_attention_dim[i],
                attn_num_head_channels=attention_head_dim[i],
                downsample_padding=downsample_padding,
                dual_cross_attention=dual_cross_attention,
                use_linear_projection=use_linear_projection,
                only_cross_attention=only_cross_attention[i],
                upcast_attention=upcast_attention,
                resnet_time_scale_shift=resnet_time_scale_shift,
                resnet_skip_time_act=resnet_skip_time_act,
                resnet_out_scale_factor=resnet_out_scale_factor,
                cross_attention_norm=cross_attention_norm,
            )
            self.down_blocks.append(down_block)

        # mid
        if mid_block_type == "UNetMidBlock2DCrossAttn":
            self.mid_block = UNetMidBlock2DCrossAttn(
                in_channels=block_out_channels[-1],
                temb_channels=blocks_time_embed_dim,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                output_scale_factor=mid_block_scale_factor,
                resnet_time_scale_shift=resnet_time_scale_shift,
                cross_attention_dim=cross_attention_dim[-1],
                attn_num_head_channels=attention_head_dim[-1],
                resnet_groups=norm_num_groups,
                dual_cross_attention=dual_cross_attention,
                use_linear_projection=use_linear_projection,
                upcast_attention=upcast_attention,
            )
        elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn":
            self.mid_block = UNetMidBlock2DSimpleCrossAttn(
                in_channels=block_out_channels[-1],
                temb_channels=blocks_time_embed_dim,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                output_scale_factor=mid_block_scale_factor,
                cross_attention_dim=cross_attention_dim[-1],
                attn_num_head_channels=attention_head_dim[-1],
                resnet_groups=norm_num_groups,
                resnet_time_scale_shift=resnet_time_scale_shift,
                skip_time_act=resnet_skip_time_act,
                only_cross_attention=mid_block_only_cross_attention,
                cross_attention_norm=cross_attention_norm,
            )
        elif mid_block_type is None:
            self.mid_block = None
        else:
            raise ValueError(f"unknown mid_block_type : {mid_block_type}")

        # count how many layers upsample the images
        self.num_upsamplers = 0

        # up
        reversed_block_out_channels = list(reversed(block_out_channels))
        reversed_attention_head_dim = list(reversed(attention_head_dim))
        reversed_layers_per_block = list(reversed(layers_per_block))
        reversed_cross_attention_dim = list(reversed(cross_attention_dim))
        only_cross_attention = list(reversed(only_cross_attention))

        output_channel = reversed_block_out_channels[0]
        for i, up_block_type in enumerate(up_block_types):
            is_final_block = i == len(block_out_channels) - 1

            prev_output_channel = output_channel
            output_channel = reversed_block_out_channels[i]
            input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)]

            # add upsample block for all BUT final layer
            if not is_final_block:
                add_upsample = True
                self.num_upsamplers += 1
            else:
                add_upsample = False

            up_block = get_up_block(
                up_block_type,
                num_layers=reversed_layers_per_block[i] + 1,
                in_channels=input_channel,
                out_channels=output_channel,
                prev_output_channel=prev_output_channel,
                temb_channels=blocks_time_embed_dim,
                add_upsample=add_upsample,
                resnet_eps=norm_eps,
                resnet_act_fn=act_fn,
                resnet_groups=norm_num_groups,
                cross_attention_dim=reversed_cross_attention_dim[i],
                attn_num_head_channels=reversed_attention_head_dim[i],
                dual_cross_attention=dual_cross_attention,
                use_linear_projection=use_linear_projection,
                only_cross_attention=only_cross_attention[i],
                upcast_attention=upcast_attention,
                resnet_time_scale_shift=resnet_time_scale_shift,
                resnet_skip_time_act=resnet_skip_time_act,
                resnet_out_scale_factor=resnet_out_scale_factor,
                cross_attention_norm=cross_attention_norm,
            )
            self.up_blocks.append(up_block)
            prev_output_channel = output_channel

        # out
        if norm_num_groups is not None:
            self.conv_norm_out = nn.GroupNorm(
                num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps
            )

            if act_fn == "swish":
                self.conv_act = lambda x: F.silu(x)
            elif act_fn == "mish":
                self.conv_act = nn.Mish()
            elif act_fn == "silu":
                self.conv_act = nn.SiLU()
            elif act_fn == "gelu":
                self.conv_act = nn.GELU()
            else:
                raise ValueError(f"Unsupported activation function: {act_fn}")

        else:
            self.conv_norm_out = None
            self.conv_act = None

        conv_out_padding = (conv_out_kernel - 1) // 2
        self.conv_out = nn.Conv2d(
            block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding
        )


import cv2

#load pre trained data
trained_face_data = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

#choose image to detect face in
#img = cv2.imread('52-05.jpg')
#img = cv2.imread('img2p.jpg')
webcam = cv2.VideoCapture(1)  #detect face in video
#key = cv2.waitKey(1)

#iterate over frames
while True:
    successful_frame_read, frame = webcam.read() #get current frame

#make it grayscale
    gray_img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    #cv2.waitKey(1)

#detect faces
    face_coordinates = trained_face_data.detectMultiScale(gray_img)
    # print(face_coordinates)

#draw rectangle around face
    for (x, y, w, h) in face_coordinates:
        cv2.rectangle(frame, (x, y), (x+w, y+h), (255, 255, 0), 3)

#show image
    cv2.imshow('face detector', frame)
    key = cv2.waitKey(1)

    #stop if q is pressed
    if key==81 or key ==113:
        break

#release the videocapture object
webcam.release()

print('code completed')


#!/usr/bin/env python3
"""
A function that uses the requests module to obtain the HTML content
of a particular URL and return it
"""

import redis
import requests
from functools import wraps

r = redis.Redis()

def url_access_count(method):
    """
    A decorator for the get_page function.
    """
    @wraps(method)
    def wrapper(url):
        """wrap decorated function"""
        key = "cached:" + url
        cached_value = r.get(key)
        if cached_value:
            return cached_value.decode("utf-8")

        key_count = "count:" + url
        html_content = method(url)

        r.incr(key_count)
        r.set(key, html_content, ex=10)
        r.expire(key, 10)
        return html_content

    return wrapper

@url_access_count
def get_page(url: str) -> str:
    """
    Obtain the HTML content, track the number of accesses,
    and cache the result with a 10-second expiration.
    """
    results = requests.get(url)

    key_count = "count:" + url
    count = r.get(key_count).decode("utf-8")
    print(count)

    return results.text

if __name__ == "__main__":
    get_page('http://google.com')

    print("OK")


# Document  : 1 HelloWorld.py
# Created on: 13-07-2019, 06:05:07 PM
# Author    : Nivesh-GC

# Udemy: Complete Python BootCamp
# https://www.youtube.com/watch?v=DjEuROpsvp4
# Setting up a Python Development Environment in Atom

print('Hello World of Atom')
a = 50
b = 50
c = a + b
d = type(c)
print(d)


import warnings
from dataclasses import dataclass
from typing import List, Optional

import keopscore
import torch
from pykeops.torch import Genred

from falkon.mmv_ops.utils import _get_gpu_info, _start_wait_processes, create_output_mat
from falkon.options import BaseOptions, FalkonOptions
from falkon.utils import decide_cuda
from falkon.utils.helpers import calc_gpu_block_sizes, sizeof_dtype
from falkon.utils.stream_utils import sync_current_stream

@dataclass(frozen=True)
class ArgsFmmv:
    X1: torch.Tensor
    X2: torch.Tensor
    v: torch.Tensor
    other_vars: List[torch.Tensor]
    out: torch.Tensor
    gpu_ram: float
    backend: str
    function: callable

def _decide_backend(opt: BaseOptions, num_dim: int) -> str:
    """Switch between CPU and GPU backend for KeOps"""
    if not decide_cuda(opt):
        return "CPU"
    else:
        return "GPU_1D"

def _estimate_split(N, M, D, T, R, ds):
    """Estimate the splits along dimensions N and M for a MVM to fit in memory

    The operations consist of computing the product between a kernel
    matrix (from a N*D and a M*D matrix) and a 'vector' of shape M*T
    This typically requires storage of the input and output matrices,
    which occupies (M + N)*(D + T) memory locations plus some intermediate
    buffers to perform computations.
    TODO: It is not clear how much intermediate memory KeOps requires;
    the only thing that is certain is that it is quadratic in D.
    For now we sidestep this issue by using a smaller R than what is
    actually available in GPU memory.

    This function calculates the split along N and M into blocks of size n*m
    so that we can compute the kernel-vector product between such blocks
    and still fit in GPU memory.

    Parameters
    -----------
     - N : int
        The first dimension of the kernel matrix
     - M : int
        The second dimension of the kernel matrix
     - D : int
        The data dimensionality
     - T : int
        The number of output columns
     - R : float
        The amount of memory available (in bytes)
     - ds : int
        The size in bytes of each element in the data matrices
        (e.g. 4 if the data is in single precision).

    Returns
    --------
     - n : int
        The block size to be used along the first dimension
     - m : int
        The block size along the second dimension of the kernel
        matrix

    Raises
    -------
    RuntimeError
        If the available memory `R` is insufficient to store even the smallest
        possible input matrices. This may happen if `D` is very large since we
        do not perform any splitting along `D`.

    Notes
    ------
    We find 'good' values of M, N such that
    N*(D+T) + M*(D+T) <= R/ds
    """
    R = R / ds

    # We have a linear equation in two variables (N, M)
    slope = -1
    intercept = R / (D + T)

    slack_points = 10
    # We try to pick a point at the edges such that only one kind of split
    # is necessary
    if N < intercept - 1:
        M = min(M, intercept + slope * N)
    elif M < intercept - 1:
        N = min(N, intercept + slope * M)
    else:
        # All points on the slope such that N, M > 0 are possible
        N = intercept - slack_points - 1
        M = intercept + slope * N

    if N <= 0 or M <= 0:
        raise RuntimeError("Insufficient available GPU memory (available %.2fGB)" % (R * ds / 2**30))

    return int(N), int(M)

def _single_gpu_method(proc_idx, queue, device_id):
    a: ArgsFmmv = queue.get()
    backend = a.backend
    X1 = a.X1
    X2 = a.X2
    v = a.v
    oout = a.out
    other_vars = a.other_vars
    fn = a.function
    R = a.gpu_ram

    N, D = X1.shape
    M = X2.shape[0]
    T = v.shape[1]
    device = torch.device(f"cuda:{device_id}")

    # Second round of subdivision (only if necessary due to RAM constraints)
    n, m = _estimate_split(N, M, D, T, R, sizeof_dtype(X1.dtype))

    other_vars_dev = [ov.to(device, copy=False) for ov in other_vars]
    out_ic = oout.device.index == device_id

    # Process the two rounds of splitting with a nested loop.
    with torch.cuda.device(device_id), torch.autograd.inference_mode():
        for mi in range(0, M, m):
            ml = min(m, M - mi)
            if ml != M and mi > 0:  # Then we must create a temporary output array
                out = torch.empty_like(oout)
            else:
                out = oout

            cX2 = X2[mi : mi + ml, :].to(device, copy=False)
            cv = v[mi : mi + ml, :].to(device, copy=False)

            for ni in range(0, N, n):
                nl = min(n, N - ni)
                cX1 = X1[ni : ni + nl, :].to(device, copy=False)
                cout = out[ni : ni + nl, :].to(device, copy=False)

                variables = [cX1, cX2, cv] + other_vars_dev
                fn(*variables, out=cout, device_id=device_id, backend=backend)
                if not out_ic:
                    out[ni : ni + nl, :].copy_(cout)
            if ml != M and mi > 0:
                oout.add_(out)

    return oout

def run_keops_mmv(
    X1: torch.Tensor,
    X2: torch.Tensor,
    v: torch.Tensor,
    other_vars: List[torch.Tensor],
    out: Optional[torch.Tensor],
    formula: str,
    aliases: List[str],
    axis: int,
    reduction: str = "Sum",
    opt: Optional[FalkonOptions] = None,
) -> torch.Tensor:
    if opt is None:
        opt = FalkonOptions()
    # Choose backend
    N, D = X1.shape
    T = v.shape[1]
    backend = _decide_backend(opt, D)
    data_devs = [X1.device, X2.device, v.device]

    if any(ddev.type == "cuda" for ddev in data_devs) and (not backend.startswith("GPU")):
        warnings.warn(
            "KeOps backend was chosen to be CPU, but GPU input tensors found. "
            "Defaulting to 'GPU_1D' backend. To force usage of the CPU backend, "
            "please pass CPU tensors; to avoid this warning if the GPU backend is "
            "desired, check your options (i.e. set 'use_cpu=False')."
        )
        backend = "GPU_1D"

    differentiable = any([X1.requires_grad, X2.requires_grad, v.requires_grad] + [o.requires_grad for o in other_vars])

    comp_dev_type = backend[:3].lower().replace("gpu", "cuda")  # 'cpu' or 'cuda'
    keopscore.config.config.use_cuda = comp_dev_type == "cuda"  # workaround for keops issue#248
    out = create_output_mat(
        out,
        data_devs,
        is_sparse=False,
        shape=(N, T),
        dtype=X1.dtype,
        comp_dev_type=comp_dev_type,
        other_mat=X1,
        output_stride="C",
    )
    rec_multVar_highdim = None
    if D > 100:
        rec_multVar_highdim = 1
    fn = Genred(
        formula,
        aliases,
        reduction_op=reduction,
        axis=axis,
        dtype_acc=opt.keops_acc_dtype,
        sum_scheme=opt.keops_sum_scheme,
        rec_multVar_highdim=rec_multVar_highdim,
    )
    if differentiable:
        # For differentiable inputs we don't split, since we don't know how to
        # split the backward pass.
        out = fn(X1, X2, v, *other_vars, out=out, backend=backend)
    elif comp_dev_type == "cpu" and all(ddev.type == "cpu" for ddev in data_devs):  # incore CPU
        out = fn(X1, X2, v, *other_vars, out=out, backend=backend)
    elif comp_dev_type == "cuda" and all(ddev.type == "cuda" for ddev in data_devs):  # incore CUDA
        device = data_devs[0]
        with torch.cuda.device(device):
            sync_current_stream(device)
            out = fn(X1, X2, v, *other_vars, out=out, backend=backend)
    else:  # cpu data, gpu computations: out-of-core
        # slack should be high due to imprecise memory usage estimates for keops
        gpu_info = _get_gpu_info(opt, slack=opt.keops_memory_slack)
        block_sizes = calc_gpu_block_sizes(gpu_info, N)

        args = []  # Arguments passed to each subprocess
        for i, g in enumerate(gpu_info):
            # First round of subdivision
            bwidth = block_sizes[i + 1] - block_sizes[i]
            if bwidth <= 0:
                continue
            args.append(
                (
                    ArgsFmmv(
                        X1=X1.narrow(0, block_sizes[i], bwidth),
                        X2=X2,
                        v=v,
                        out=out.narrow(0, block_sizes[i], bwidth),
                        other_vars=other_vars,
                        function=fn,
                        backend=backend,
                        gpu_ram=g.usable_memory,
                    ),
                    g.Id,
                )
            )
        _start_wait_processes(_single_gpu_method, args)

    return out


import os
import shutil
import sys

import pytest
import torch

from ivory.core.client import create_client

@pytest.fixture(scope="module")
def runs():
    sys.path.insert(0, os.path.abspath("examples"))
    client = create_client(directory="examples")
    runs = []
    for name in ["tensorflow", "nnabla", "torch2"]:
        run = client.create_run(name, epochs=5, batch_size=10, shuffle=False)
        runs.append(run)
    run_tf, run_nn, run_torch = runs

    run_nn.model.build(
        run_nn.trainer.loss, run_nn.datasets.train, run_nn.trainer.batch_size
    )
    run_nn.optimizer.set_parameters(run_nn.model.parameters())

    ws_tf = run_tf.model.weights
    ws_nn = run_nn.model.parameters().values()
    ws_torch = run_torch.model.parameters()
    for w_tf, w_nn, w_torch in zip(ws_tf, ws_nn, ws_torch):
        w_nn.data.data = w_tf.numpy()
        w_torch.data = torch.tensor(w_tf.numpy().T)

    yield dict(zip(["tf", "nn", "torch"], runs))
    del sys.path[0]
    if os.path.exists("examples/mlruns"):
        shutil.rmtree("examples/mlruns")


from unittest import TestCase
import glob

from lxml import etree

class ValidationError(Exception):
    pass

class TestSampleFileValidation(TestCase):
    def test_ukrdc_sample_files(self):
        # For each sample file
        for sample_path in glob.glob("sample_files/ukrdc/*.xml"):
            # Run as a subtest
            with self.subTest(msg=sample_path):
                # Open the schema and sample files for reading
                with open(
                    "schema/ukrdc/UKRDC.xsd", "r", encoding="utf-8"
                ) as schema_file, open(
                    sample_path, "r", encoding="utf-8"
                ) as sample_file:

                    # Create a new schema object to track errors for this file
                    xml_schema = etree.XMLSchema(
                        etree.parse(
                            schema_file,
                            parser=None,
                        )
                    )

                    # Try validating the sample file against the schema
                    try:
                        xml_doc = etree.parse(sample_file, None)
                        xml_schema.assertValid(xml_doc)
                    # Initially catch errors to allow reporting multiple issues in one file
                    except etree.DocumentInvalid as e:
                        tree = etree.ElementTree(xml_doc.getroot())
                        # Print all errors
                        print("Validation error(s):")
                        for error in xml_schema.error_log:
                            print("  Line {}: {}".format(error.line, error.message))

                            for e in tree.xpath(".//*"):
                                if error.line == e.sourceline:
                                    xml_path = tree.getpath(e)
                                    print(xml_path)
                                    break

                        # Raise an exception to fail the test and report the full error list
                        raise ValidationError(
                            f"{len(xml_schema.error_log)} validation error(s) in {sample_path}. See full output above for details."
                        ) from e


from django.conf.urls import url
from . import views

from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    url(r'^$', views.first_view, name='first_view'),
    url(r'^uimage/$', views.uimage, name='uimage'),
    url(r'^dface/$', views.dface, name='dface'),
    url(r'^crop/$', views.crop, name='crop'),
    url(r'^backgroundsubtract/$', views.backgroundsubtract, name='backgroundsubtract'),
    url(r'^binarize/$', views.binarize, name='binarize'),
    url(r'^webcam/$', views.webcam, name='webcam'),
    url(r'^stream/$', views.stream, name='stream'),
    url(r'^capture/$', views.capture, name='capture'),
]

urlpatterns += static(settings.MEDIA_URL,
                      document_root=settings.MEDIA_ROOT)


from urllib.parse import quote_plus
from bs4 import BeautifulSoup
#selenium : web test에 사용되는 프레임워크, webdriver API를 통해 렌더링이 완료된 후의 DOM 결과물에 접근할 수 있음(브라우저 제어가 필요)
#pip install selenium
#직접 브라우저를 제어하기 때문에 header값 없이도 크롤링이 가능
#여기선 Chrome 사용 webdriver 설치 : https://chromedriver.chromium.org/downloads
from selenium import webdriver

baseUrl = 'https://www.google.com/search?q='
plusUrl = input('검색어 입력 : ')
resultUrl = baseUrl + quote_plus(plusUrl)

#chrome webDriver 위치가 현재 개발폴더 위치와 다르면 Chrome({경로})와 같이 사용
driver = webdriver.Chrome()

#브라우저가 열리고 입력된 url로 이동
driver.get(resultUrl)

html = driver.page_source
soup = BeautifulSoup(html)

#select로 가져올 경우 list형식으로 가져옴
r = soup.select('.r')
for i in r:
    #list object의 경우엔 text를 가져올 수 없음, 텍스트를 불러오기 위해 select_on 사용
    print(i.select_one('.LC20lb.DKV0Md').text)
    #print(i.select_one('.iUh30.bc').text)
    print(i.a.attrs['href'], '\n')

driver.close()


import Classes as Cls
import scr.FormatFunctions as Format
import scr.SamplePathClasses as PathCls
import scr.FigureSupport as Figs
import scr.StatisticalClasses as Stat
import P1 as P1

alpha = P1.alpha

#Find comparative outcome
def get_compare(sim_output_fair, sim_output_unfair):
    increase = Stat.DifferenceStatIndp(
        name='Increase in Reward',
        x=sim_output_fair.get_rewards(),
        y_ref=sim_output_unfair.get_rewards()
    )
    # estimate and CI
    estimate_CI = Format.format_estimate_interval(
        estimate=increase.get_mean(),
        interval=increase.get_t_CI(alpha=P1.alpha),
        deci=1
    )
    print("Average increase in survival time (years) and {:.{prec}%} confidence interval:".format(1 - P1.alpha, prec=0),
          estimate_CI)


from django.core.mail import send_mail
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.db.models import Count
from django.shortcuts import render, get_object_or_404, redirect
from django.views.decorators.http import require_POST
from django.views.generic import ListView
from taggit.models import Tag
from django.contrib.postgres.search import SearchVector
from django.contrib import messages

import djangoProject.settings
from .forms import EmailPostForm, CommentForm, SearchForm
from .models import Post

class PostListView(ListView):
    """
     Альтернативное представление списка постов
    """

    # Вместо
    # определения атрибута queryset мы могли бы указать model=Post, и Django
    # сформировал бы для нас типовой набор запросов Post.objects.all()
    queryset = Post.published.all()
    context_object_name = 'posts'
    paginate_by = 3
    template_name = 'blog/post/list.html'

def post_list(request, tag_slug=None):
    post_list = Post.published.all()
    tag = None
    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        post_list = post_list.filter(tags__in=[tag])
    # создаем объект класс Paginator  с числом объектов на 1 странице
    paginator = Paginator(post_list, 3)
    # вытягиваем значение параметра page из GET запроса, если он отсутствует, выставляем дефолтное 1
    #     MultiValueDict????
    page_number = request.GET.get('page', 1)
    # получаем объекты для указанной страницы
    try:
        posts = paginator.page(page_number)
    except EmptyPage:
        posts = paginator.page(1)
    except PageNotAnInteger:
        posts = paginator.page(paginator.num_pages)

    print(posts.__dict__)
    return render(request, 'blog/post/list.html', {'posts': posts,
                                                   'tag': tag})

def post_detail(request, post, year, month, day):
    print('мы тут с ', request.user)
    post = get_object_or_404(Post,
                             status=Post.Status.PUBLISHED,
                             slug=post,
                             publish__year=year,
                             publish__month=month,
                             publish__day=day)
    # Список активных комментариев к этому посту
    comments = post.comments.filter(active=True)
    # Форма для комментирования пользователями
    form = CommentForm()

    # список схожих постов
    # values_list возвращает кортеж со значением заданных полей
    post_tags_ids = post.tags.values_list('id', flat=True)
    # модификатор __in -- значение должно быть в указанном списке кортоже квересете
    similar_posts = Post.published.filter(tags__in=post_tags_ids).exclude(id=post.id)
    # annotate создает переменную в который хранит результат
    # агрегированного выражения над 'tag'
    # названием переменной выступает ключ -- same_tags
    similar_posts = similar_posts.annotate(same_tags=Count('tags')).order_by('-same_tags', '-publish')[:4]

    return render(request, 'blog/post/details.html', {'post': post,
                                                      'comments': comments,
                                                      'form': form,
                                                      'similar_posts': similar_posts})

# представлени для 1)отображает изначальные данные на странице
#                  2)обработка представленных для валидации данных
def post_share(request, post_id):
    # функция сокращенного доступа для извлечения поста с id==post_id
    post = get_object_or_404(Post,
                             id=post_id,
                             status=Post.Status.PUBLISHED)
    sent = False
    # 2)
    if request.method == "POST":
        #  Когда пользователь заполняет форму и  передает ее методом POST
        form = EmailPostForm(request.POST)
        if form.is_valid():
            # форма снова прорисовывается в  шаблоне,
            # включая переданные данные. Ошибки валидации будут отображены
            # в шаблоне.
            # в cd = dict  в котором  находятся данные из формы,
            # где ключи =  названия формы и значение = содержание
            # form.cleaned_data returns a dictionary of validated form input fields and their values, where string primary keys are returned as objects.
            #
            # form.data returns a dictionary of un-validated form input fields and their values in string format (i.e. not objects).
            cd = form.cleaned_data
            # непосредственно отправка письма
            post_url = request.build_absolute_uri(
                post.get_absolute_url()
            )
            subject = f"{cd['name']} recommends you read {post}"
            message = f"Mail send by {cd['email']}\n\n" \
                      f"Read {post.title} at {post_url}\n\n" \
                      f"{cd['name']}\'s comments: {cd['comments']}"
            send_mail(subject, message, djangoProject.settings.EMAIL_HOST_USER,
                      [cd['to']])
            sent = True
    # 1)
    else:
        # Указанный экземпляр формы
        # будет использоваться для отображения пустой формы в шаблоне
        form = EmailPostForm()
    return render(request, 'blog/post/share.html', {'post': post,
                                                    'form': form,
                                                    'sent': sent})

@require_POST
def post_comment(request, post_id):
    post = get_object_or_404(Post,
                             id=post_id,
                             status=Post.Status.PUBLISHED)
    comment = None
    #  Создается экземпляр формы, используя переданные на обработку POSTданные

    form = CommentForm(data=request.POST)
    if form.is_valid():
        # Метод save() создает экземпляр модели, к которой форма привязана,
        # и сохраняет его в базе данных. Если вызывать его, используя commit=False,
        # то экземпляр модели создается, но не сохраняется в базе данных. Такой
        # подход позволяет видоизменять объект перед его окончательным сохранением.

        comment = form.save(commit=False)
        print(comment.__dict__)
        comment.post = post
        comment.save()
    return render(request, 'blog/post/comment.html',
                  {'post': post,
                   'form': form,
                   'comment': comment})

def post_search(request):
    form = SearchForm()
    query = None
    results = []
    if 'query' in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            # cleaned_data -- словарь, который хранит информацию из формы, прошедшую валидацию
            query = form.cleaned_data['query']
            results = Post.published.annotate(search=SearchVector('title', 'body')
                                              ).filter(search=query)
    return render(request,
                  'blog/post/search.html',
                  {'form': form,
                   'query': query,
                   'results': results})

# создал вьюшку редирект на главную страницу если введен некорректный url
def redir_to_main_page(request, id):
    messages.add_message(request, messages.INFO, 'you were redirected on main page')
    return redirect('blog:post_list')


import sqlite3, sys
from pathlib import Path
from . import Notes
from tqdm import tqdm

db_path = "database/xhs_tesla_notes.db"

def fill(db_path=db_path):
    blank_query = "SELECT COUNT(*) FROM notes WHERE content is ''"
    try:
        conn = sqlite3.connect(db_path)
        cursor = conn.cursor()

        amount = cursor.execute("SELECT COUNT(*) FROM notes").fetchone()[0]
        blank_amount = cursor.execute(blank_query).fetchone()[0]
        print(
            f"There are {amount} notes in the database, {blank_amount} of them have blank content, blank rate is {blank_amount/amount}"
        )

        blank_notes_query = cursor.execute(blank_query)
        notes_null_content = blank_notes_query.fetchall()
        # notes_null_content = []

        for note in tqdm(notes_null_content):
            try:
                null_con = Notes.Note(
                    note[0], note[1], note[2], note[3], note[4], note[5], note[10]
                )
                print(f'Filling content for note with id {null_con.id}')
                content = null_con.get_content()
                # print(f"Obtained content: {content}")

                update_query = "UPDATE notes SET content = ? WHERE id = ?"
                cursor.execute(update_query, (content, null_con.id))
                conn.commit()
                # print(f'Successfully filled content for note with id {null_con.id}')

            except Exception as inner_exc:
                print(f"Error processing note with id {note[0]}: {inner_exc}")

        New_blank_amount = cursor.execute(blank_query).fetchone()[0]
        print(
            f"{blank_amount-New_blank_amount} of them have been filled, blank_rate now is {New_blank_amount/amount} as {New_blank_amount} of {amount}"
        )

    except Exception as outer_exc:
        print(f"Error connecting to the database: {outer_exc}")

    finally:
        conn.commit()
        conn.close()


from dataclasses import dataclass, asdict, field, make_dataclass
from typing import List, Union, Any, Dict
from enum import Enum
from sigrok.core.classes import ConfigKey

__all__ = [
    "SrBlankState",
    "SrDeviceState",
    #"SrFileState",
    #"AnalogChannel",
    #"LogicChannel",
]

colorsArray = [
    '#fce94f', '#fcaf3e', '#e9b96e', '#8ae234', '#729fcf', '#ad7fa8', '#cf72c3', '#ef2929',
    '#edd400', '#f57900', '#c17d11', '#73d216', '#3465a4', '#75507b', '#a33496', '#cc0000',
    '#c4a000', '#ce5c00', '#8f5902', '#4e9a06', '#204a87', '#5c3566', '#87207a', '#a40000',
    '#16191a', '#2e3436', '#555753', '#888a8f', '#babdb6', '#d3d7cf', '#eeeeec', '#ffffff'
]

def factory(data):
    return dict(x for x in data if x[1] is not None)

def build_opts(opts: list):
    #NOTE: key, id, name, desc, keyName, caps
    data = []
    for opt in opts:
        bases = []
        opt_fields = []
        if 'caps' in opt:
            if 'LIST' in opt['caps']:
                bases.append(ConfList)
            elif 'GET' in opt['caps']:
                bases.append(ConfValue)

        for k, v in opt.items():
            opt_fields.append((k, type(v)))

        opt_fields.append( ('keyName', str))

        keyName=ConfigKey.get(opt['key']).name

        SrOpt = make_dataclass(cls_name='SrOpt', fields=opt_fields, bases=tuple(bases))
        srOpt = SrOpt(**opt, keyName = keyName)

        data.append( (keyName, srOpt) )

    return data

@dataclass
class ConfValue:
    value: Any

@dataclass
class ConfList:
    values: List[Any] = field(default_factory=list)

@dataclass
class Channel:
    name: str
    text: str
    color: str
    enabled: bool
    index: int
    type: str

    def update(self, opts: Dict):
        for key, value in opts.items():
            if hasattr(self, key):
                setattr(self, key, value)

@dataclass
class LogicChannel(Channel):
    traceHeight: int = 34

@dataclass
class AnalogChannel(Channel):
    pVertDivs: int = 1
    nVertDivs: int = 1
    divHeight: int = 51
    vRes: float = 20.0
    autoranging: bool = True
    conversion: str = ''
    convThres: str = ''
    showTraces: str = ''

class ChTypesEnum(Enum):
    analog = AnalogChannel
    logic = LogicChannel

@dataclass
class ChnlsContBase:
    def set(self, chOpts: List[Dict] ):
        result = []
        for opts in chOpts:
            chName = opts.pop('chName', None)
            if hasattr(self, chName):
                attr = getattr(self, chName)
                attr.update(opts)
                result.append(chName)
        return result

    def get(self):
        data = asdict(self, dict_factory=factory)
        return data.values()

    def get_list(self):
        data = asdict(self, dict_factory=factory)
        return list( { item['type'] for item in data.values() })

@dataclass
class OptsContBase:
    def set(self, opts: List[Dict] ):
        result = []
        for item in opts:
            optName = item['keyName']
            if hasattr(self, optName):
                attr = getattr(self, optName)
                attr.value = item['value']
                result.append(optName)
        return result

    def get(self):
        return asdict(self, dict_factory=factory)

    def get_list(self):
        data = [ item['key'] for item in asdict(self).values() ] #list(asdict(self).values()
        return  data

def make_ch_container(channels: List[Dict]):
    fields = []
    for i, item in enumerate(channels):
        chInst = ChTypesEnum[item['type']].value(**item, color= colorsArray[i], text=item['name'])
        fields.append( ( item['name'], chInst) )

    ChnlsContainer = make_dataclass('ChnlsContainer', [ item[0] for item in fields], bases=tuple([ChnlsContBase]) )
    data = ChnlsContainer(**{ item[0]: item[1] for item in fields})
    return data

def make_opts_container(opts: List[Dict]):
    fields = build_opts(opts)
    OptsContainer = make_dataclass('OptsContainer', [ item[0] for item in fields], bases=tuple([OptsContBase]))
    data = OptsContainer(**{ item[0]: item[1] for item in fields})
    return data

#-----------------  STATE TYPES   -----------------#
@dataclass
class SrBlankState:
    id: str
    name: str
    sourcename: str
    type: str = field(init=False)

    def __post_init__(self):
        self.type = "BLANK"

    def get(self):
        data = asdict(self, dict_factory=factory)
        return data

    def copy(self):
        return self.__class__(self.get())

@dataclass
class SrDeviceState(SrBlankState):
    drvopts: Dict[ str, Any ]
    devopts: Dict[ str, Any ]
    channels: Dict[ str, Union[ AnalogChannel, LogicChannel ] ]#Any

    def __init__(self, id, name, sourcename, drvopts: List, devopts: List, channels: List[ChTypesEnum]):
        SrBlankState.__init__(self, id, name, sourcename)
        self.type = 'DEVICE'
        self.drvopts = make_opts_container(drvopts)
        self.devopts = make_opts_container(devopts)

        self.channels = make_ch_container(channels)

    def get(self):
        data = super().get()

        data['drvopts'] = self.drvopts.get_list()
        data['devopts'] = self.devopts.get_list()
        data['channels'] = self.channels.get_list()

        return data


from flask import Flask
from flask_cors import CORS
from flask_marshmallow import Marshmallow
from config import config
from .main import main as main_blueprint

'''
Application factory for application package. \
Delays creation of an app by moving it into a factory function that can be \
explicitly invoked from script and apply configuration changes.
'''

cors = CORS(
    main_blueprint,
    origins=['http://127.0.0.1:4200', 'http://localhost:4200'],
    supports_credentials=True
)

ma = Marshmallow()

def create_app(config_name):
    app = Flask(__name__)
    # Importing configuration settings directly into app
    app.config.from_object(config[config_name])
    config[config_name].init_app(app)

    # Initializing extensions after app is created
    cors.init_app(app)
    ma.init_app(app)

    # Manually creating app_context to access objects outside of view functions
    with app.app_context():
        app.register_blueprint(main_blueprint, url_prefix='/daron')

    return app


#import mxnet.ndarray as nd
from mxnet import nd
from mxnet import autograd

# REF [site] >> https://gluon-crash-course.mxnet.io/ndarray.html
def ndarray_example():
	a = nd.array(((1, 2, 3), (5, 6, 7)))
	b = nd.full((2, 3), 2.0)
	b.shape, b.size, b.dtype

	# Operations.
	x = nd.ones((2, 3))
	y = nd.random.uniform(-1, 1, (2, 3))
	x * y
	y.exp()
	nd.dot(x, y.T)

	# Indexing.
	y[1, 2]
	y[:, 1:3]
	y[:, 1:3] = 2
	y[1:2, 0:2] = 4

	# Converting between MXNet NDArray and NumPy.
	na = x.asnumpy()
	nd.array(na)

# REF [site] >> https://gluon-crash-course.mxnet.io/autograd.html
def autograd_example():
	# When differentiating a function f(x)=2x2 with respect to parameter x.
	x = nd.array([[1, 2], [3, 4]])
	x.attach_grad()

	# To let MXNet store y, so that we can compute gradients later, we need to put the definition inside a autograd.record() scope.
	with autograd.record():
		y = 2 * x * x

	# Invoke back propagation (backprop).
	#	When y has more than one entry, y.backward() is equivalent to y.sum().backward().
	y.backward()

	print('x.grad =', x.grad)

	# Using Python control flows.
	def f(a):
		b = a * 2
		while b.norm().asscalar() < 1000:
			b = b * 2
		if b.sum().asscalar() >= 0:
			c = b[0]
		else:
			c = b[1]
		return c

	a = nd.random.uniform(shape=2)
	a.attach_grad()

	with autograd.record():
		c = f(a)
	c.backward()

def main():
	ndarray_example()
	autograd_example()

#---------------------------------------------------------------------

if '__main__' == __name__:
	main()


import sys
import logging
import click
import os
sys.path.append('.')
from src.classes import Dataset

logger = logging.getLogger(__name__)

@click.command()
@click.option(
    '--n_images', default=10,
    help="Number of images per tissue"
)
@click.option(
    '--n_tissues', default=6,
    help="Number of tissues with most numbers of samples"
)
def main(n_images, n_tissues):
    os.makedirs('data/patches', exist_ok=True)
    logger.info('Initializing patches script')
    dataset = Dataset(n_images=n_images, n_tissues=n_tissues)
    dataset.get_patchcoordfiles()

if __name__ == '__main__':
    logging.basicConfig(
        filename='logs/patches.log', level=logging.DEBUG,
        format=(
            "%(asctime)s | %(name)s | %(processName)s |"
            "%(levelname)s: %(message)s"
        )
    )
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('spurt', '0020_linkpost_scrape_token'),
    ]

    operations = [
        migrations.RenameField(
            model_name='linkpost',
            old_name='pub_date',
            new_name='scraped_pub_date',
        ),
    ]


from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.http import MediaFileUpload
import subprocess
import os
from os.path import join

path = os.getcwd()

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/drive']

def main():
    """Shows basic usage of the Drive v3 API.
    Prints the names and ids of the first 10 files the user has access to.
    """
    # Backup the tweets
    subprocess.call(['tar -czvf tweet.tar.gz /usr/local/airflow/data/', '-1'], shell=True)

    creds = None
    # The file token.pickle stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    print (join(path,'dags/daglibs/token.pickle'))
    if os.path.exists(join(path,'dags/daglibs/token.pickle')):
        with open(join(path,'dags/daglibs/token.pickle'), 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(join(path,
                'dags/daglibs/credentials.json'), SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open(join(path,'dags/daglibs/token.pickle'), 'wb') as token:
            pickle.dump(creds, token)

    service = build('drive', 'v3', credentials=creds)

    # Call the Drive v3 API
    file_metadata = {'name': 'tweet.tar.gz'}
    media = MediaFileUpload('/usr/local/airflow/tweet.tar.gz', mimetype='*/*')
    file = service.files().create(body=file_metadata,
                                        media_body=media,
                                        fields='id').execute()
    print ("File ID: {}".format(file.get('id')))
    if file.get('id'):
        return True
    return False

if __name__ == '__main__':
    main()


import collections

from oneview_redfish_toolkit.api.errors import \
    OneViewRedfishException
from oneview_redfish_toolkit.api.errors import \
    OneViewRedfishResourceNotFoundException
from oneview_redfish_toolkit.api.redfish_json_validator import \
    RedfishJsonValidator
from oneview_redfish_toolkit import config

class RedfishError(RedfishJsonValidator):
    """Creates a Redfish Error Dict

        Populates self.redfish with errors. Will not validate as there's no
        schema to validate against.

    """

    SCHEMA_NAME = None

    def __init__(self, code, message):
        """Constructor

            Populates self.redfish with error message.
        """

        super().__init__(self.SCHEMA_NAME)
        self.redfish["error"] = collections.OrderedDict()
        # Check if Code is a valid Code Error in the registry
        if code not in config.get_registry_dict()["Base"]["Messages"]:
            raise OneViewRedfishResourceNotFoundException(
                "Registry {} not found.".format(code)
            )

        self.redfish["error"]["code"] = "Base.1.1." + code
        self.redfish["error"]["message"] = message
        self.redfish["error"]["@Message.ExtendedInfo"] = list()

    def add_extended_info(
        self,
        message_id,
        message_args=[],
        related_properties=[]):
        """Adds an item to ExtendedInfo list using values from DMTF registry

            Adds an item to ExtendedInfo list using the values for Message,
            Severity and Resolution from DMTF Base Registry.

            Parameters:
                message_id: Id of the message; oneOf the keys in Redfish
                    Registry Messages
                message_args: List of string to replace markers on Redfish
                    messages. Must have the same length as the number of %
                    signs found in the registry Message field
                related_properties: Properties relates to this e error if
                    necessary

        """
        messages = config.get_registry_dict()["Base"]["Messages"]

        # Verify if message_id exists in registry
        try:
            severity = messages[message_id]["Severity"]
        except Exception:
            raise OneViewRedfishResourceNotFoundException(
                "Message id {} not found.".format(message_id)
            )

        message = messages[message_id]["Message"]

        # Check if numbers of replacements and message_args length match
        replaces = message.count('%')
        replacements = len(message_args)
        if replaces != replacements:
            raise OneViewRedfishException(
                'Message has {} replacements to be made but {} args '
                'where sent'.format(replaces, replacements)
            )

        # Replacing the marks in the message. A better way to do this
        # is welcome.
        for i in range(replaces):
            message = message.replace('%' + str(i + 1), message_args[i])

        # Construct the dict
        extended_info = collections.OrderedDict()
        extended_info["@odata.type"] = "#Message.v1_0_5.Message"
        extended_info["MessageId"] = "Base.1.1." + message_id
        extended_info["Message"] = message
        extended_info["RelatedProperties"] = related_properties
        extended_info["MessageArgs"] = message_args
        extended_info["Severity"] = severity
        extended_info["Resolution"] = messages[message_id]["Resolution"]

        # Append it to the list
        self.redfish["error"]["@Message.ExtendedInfo"].append(extended_info)


import json
import logging
import threading
import traceback
from enum import Enum
from typing import Any, Dict, List, Optional, Set, Tuple

from lte.protos.policydb_pb2 import FlowMatch
from magma.common.redis.client import get_default_client
from magma.configuration.service_configs import load_service_config
from magma.pipelined.qos.qos_meter_impl import MeterManager
from magma.pipelined.qos.qos_tc_impl import TCManager, TrafficClass
from magma.pipelined.qos.types import (
    QosInfo,
    get_data,
    get_data_json,
    get_key,
    get_key_json,
    get_subscriber_data,
    get_subscriber_key,
)
from magma.pipelined.qos.utils import QosStore
from redis import ConnectionError  # pylint: disable=redefined-builtin

LOG = logging.getLogger("pipelined.qos.common")
# LOG.setLevel(logging.DEBUG)

def normalize_imsi(imsi: str) -> str:
    imsi = imsi.lower()
    if imsi.startswith("imsi"):
        imsi = imsi[4:]
    return imsi

class QosImplType(Enum):
    LINUX_TC = "linux_tc"
    OVS_METER = "ovs_meter"

    @staticmethod
    def list():
        return list(map(lambda t: t.value, QosImplType))

class SubscriberSession(object):
    def __init__(self, ip_addr: str):
        self.ip_addr = ip_addr
        self.ambr_ul = 0
        self.ambr_ul_leaf = 0
        self.ambr_dl = 0
        self.ambr_dl_leaf = 0

        self.rules: Set = set()

    def set_ambr(self, d: FlowMatch.Direction, root: int, leaf: int) -> None:
        if d == FlowMatch.UPLINK:
            self.ambr_ul = root
            self.ambr_ul_leaf = leaf
        else:
            self.ambr_dl = root
            self.ambr_dl_leaf = leaf

    def get_ambr(self, d: FlowMatch.Direction) -> int:
        if d == FlowMatch.UPLINK:
            return self.ambr_ul
        return self.ambr_dl

    def get_ambr_leaf(self, d: FlowMatch.Direction) -> int:
        if d == FlowMatch.UPLINK:
            return self.ambr_ul_leaf
        return self.ambr_dl_leaf

class SubscriberState(object):
    """
     Map subscriber -> sessions
    """

    def __init__(self, imsi: str, qos_store: Dict):
        self.imsi = imsi
        self.rules: Dict = {}  # rule -> qos handle
        self.sessions: Dict = {}  # IP -> [sessions(ip, qos_argumets, rule_no), ...]
        self._redis_store = qos_store

    def check_empty(self) -> bool:
        return not self.rules and not self.sessions

    def get_or_create_session(self, ip_addr: str):
        session = self.sessions.get(ip_addr)
        if not session:
            session = SubscriberSession(ip_addr)
            self.sessions[ip_addr] = session
        return session

    def remove_session(self, ip_addr: str) -> None:
        del self.sessions[ip_addr]

    def find_session_with_rule(self, rule_num: int) -> Optional[SubscriberSession]:
        for session in self.sessions.values():
            if rule_num in session.rules:
                return session
        return None

    def _update_rules_map(
        self, ip_addr: str, rule_num: int, d: FlowMatch.Direction,
        qos_data,
    ) -> None:

        if rule_num not in self.rules:
            self.rules[rule_num] = []

        session = self.get_or_create_session(ip_addr)
        session.rules.add(rule_num)
        self.rules[rule_num].append((d, qos_data))

    def update_rule(
        self, ip_addr: str, rule_num: int, d: FlowMatch.Direction,
        qos_handle: int, ambr: int, leaf: int,
    ) -> None:
        k = get_key_json(get_subscriber_key(self.imsi, ip_addr, rule_num, d))
        qos_data = get_data_json(get_subscriber_data(qos_handle, ambr, leaf))
        LOG.debug("Update: %s -> %s", k, qos_data)
        self._redis_store[k] = qos_data

        self._update_rules_map(ip_addr, rule_num, d, qos_data)

    def remove_rule(self, rule_num: int) -> None:
        session_with_rule = self.find_session_with_rule(rule_num)
        if session_with_rule:
            for (d, _) in self.rules[rule_num]:
                k = get_subscriber_key(
                    self.imsi, session_with_rule.ip_addr,
                    rule_num, d,
                )
                if get_key_json(k) in self._redis_store:
                    del self._redis_store[get_key_json(k)]
            session_with_rule.rules.remove(rule_num)

        del self.rules[rule_num]

    def find_rule(self, rule_num: int):
        return self.rules.get(rule_num)

    def get_all_rules(self) -> Dict[int, List[Tuple[Any, str]]]:
        return self.rules

    def get_all_empty_sessions(self) -> List:
        return [s for s in self.sessions.values() if not s.rules]

    def get_qos_handle(self, rule_num: int, direction: FlowMatch.Direction) -> int:
        rule = self.rules.get(rule_num)
        if rule:
            for d, qos_data in rule:
                if d == direction:
                    _, qid, _, _ = get_data(qos_data)
                    return qid
        return 0

class QosManager(object):
    """
    Qos Manager -> add/remove subscriber qos
    """

    def init_impl(self, datapath):
        """
        Takese in datapath, and initializes appropriate QoS manager based on config
        """
        if not self._qos_enabled:
            return
        if self._initialized:
            return

        try:
            impl_type = QosImplType(self._config["qos"]["impl"])

            if impl_type == QosImplType.OVS_METER:
                self.impl = MeterManager(datapath, self._loop, self._config)
            else:
                self.impl = TCManager(datapath, self._config)
            self.setup()
        except ValueError:
            LOG.error("%s is not a valid qos impl type", impl_type)
            raise

    @classmethod
    def debug(cls, _, __, ___):
        config = load_service_config('pipelined')
        qos_impl_type = QosImplType(config["qos"]["impl"])
        qos_store = QosStore(cls.__name__, client=get_default_client())
        for k, v in qos_store.items():
            _, imsi, ip_addr, rule_num, d = get_key(k)
            _, qid, ambr, leaf = get_data(v)
            print('imsi :', imsi)
            print('ip_addr :', ip_addr)
            print('rule_num :', rule_num)
            print('direction :', d)
            print('qos_handle:', qid)
            print('qos_handle_ambr:', ambr)
            print('qos_handle_ambr_leaf:', leaf)
            if qos_impl_type == QosImplType.OVS_METER:
                MeterManager.dump_meter_state(v)
            else:
                dev = config['nat_iface'] if d == FlowMatch.UPLINK else 'gtpu_sys_2152'
                print("Dev: ", dev)
                TrafficClass.dump_class_state(dev, qid)
                if leaf and leaf != qid:
                    print("Leaf:")
                    TrafficClass.dump_class_state(dev, leaf)
                if ambr:
                    print("AMBR (parent):")
                    TrafficClass.dump_class_state(dev, ambr)

        if qos_impl_type == QosImplType.LINUX_TC:
            dev = config['nat_iface']
            print("Root stats for: ", dev)
            TrafficClass.dump_root_class_stats(dev)
            dev = 'gtpu_sys_2152'
            print("Root stats for: ", dev)
            TrafficClass.dump_root_class_stats(dev)

    def _is_redis_available(self):
        try:
            self._redis_store.client.ping()
        except ConnectionError:
            return False
        return True

    def __init__(self, loop, config, client=get_default_client()):
        self._initialized = False
        self._clean_restart = config["clean_restart"]
        self._subscriber_state = {}
        self._loop = loop
        self._redis_conn_retry_secs = 1
        self._config = config
        # protect QoS object create and delete across a QoSManager Object.
        self._lock = threading.Lock()
        if 'qos' not in config.keys():
            LOG.error("qos field not provided in config")
            return
        self._qos_enabled = config["qos"]["enable"]
        if not self._qos_enabled:
            return
        self._apn_ambr_enabled = config["qos"].get("apn_ambr_enabled", True)
        LOG.info("QoS: apn_ambr_enabled: %s", self._apn_ambr_enabled)
        self._redis_store = QosStore(self.__class__.__name__, client)
        self.impl = None

    def setup(self):
        with self._lock:
            if not self._qos_enabled:
                return

            if self._is_redis_available():
                self._setupInternal()
            else:
                LOG.info(
                    "failed to connect to redis..retrying in %d secs",
                    self._redis_conn_retry_secs,
                )
                self._loop.call_later(self._redis_conn_retry_secs, self.setup)

    def _setupInternal(self):
        if self._initialized:
            return
        if self._clean_restart:
            LOG.info("Qos Setup: clean start")
            self.impl.setup()
            self.impl.destroy()
            self._redis_store.clear()
            self._initialized = True
        else:
            # read existing state from qos_impl
            LOG.info("Qos Setup: recovering existing state")
            self.impl.setup()

            cur_qos_state, apn_qid_list = self.impl.read_all_state()
            LOG.debug("Initial qos_state -> %s", json.dumps(cur_qos_state, indent=1))
            LOG.debug("apn_qid_list -> %s", apn_qid_list)
            LOG.debug("Redis state: %s", self._redis_store)
            try:
                # populate state from db
                in_store_qid = set()
                in_store_ambr_qid = set()
                purge_store_set = set()
                for rule, sub_data in self._redis_store.items():
                    _, qid, ambr, leaf = get_data(sub_data)
                    if qid not in cur_qos_state:
                        LOG.warning("missing qid: %s in TC", qid)
                        purge_store_set.add(rule)
                        continue
                    if ambr and ambr != 0 and ambr not in cur_qos_state:
                        purge_store_set.add(rule)
                        LOG.warning("missing ambr class: %s of qid %d", ambr, qid)
                        continue
                    if leaf and leaf != 0 and leaf not in cur_qos_state:
                        purge_store_set.add(rule)
                        LOG.warning("missing leaf class: %s of qid %d", leaf, qid)
                        continue

                    if ambr:
                        qid_state = cur_qos_state[qid]
                        if qid_state['ambr_qid'] != ambr:
                            purge_store_set.add(rule)
                            LOG.warning("Inconsistent amber class: %s of qid %d", qid_state['ambr_qid'], ambr)
                            continue

                    in_store_qid.add(qid)
                    if ambr:
                        in_store_qid.add(ambr)
                        in_store_ambr_qid.add(ambr)
                    in_store_qid.add(leaf)

                    _, imsi, ip_addr, rule_num, direction = get_key(rule)

                    subscriber = self._get_or_create_subscriber(imsi)
                    subscriber.update_rule(ip_addr, rule_num, direction, qid, ambr, leaf)
                    session = subscriber.get_or_create_session(ip_addr)
                    session.set_ambr(direction, ambr, leaf)

                # purge entries from qos_store
                for rule in purge_store_set:
                    LOG.debug("purging qos_store entry %s", rule)
                    del self._redis_store[rule]

                # purge unreferenced qos configs from system
                # Step 1. Delete child nodes
                lost_and_found_apn_list = set()
                for qos_handle in cur_qos_state:
                    if qos_handle not in in_store_qid:
                        if qos_handle in apn_qid_list:
                            lost_and_found_apn_list.add(qos_handle)
                        else:
                            LOG.debug("removing qos_handle %d", qos_handle)
                            self.impl.remove_qos(
                                qos_handle,
                                cur_qos_state[qos_handle]['direction'],
                                recovery_mode=True,
                            )

                if len(lost_and_found_apn_list) > 0:
                    # Step 2. delete qos ambr without any leaf nodes
                    for qos_handle in lost_and_found_apn_list:
                        if qos_handle not in in_store_ambr_qid:
                            LOG.debug("removing apn qos_handle %d", qos_handle)
                            self.impl.remove_qos(
                                qos_handle,
                                cur_qos_state[qos_handle]['direction'],
                                recovery_mode=True,
                                skip_filter=True,
                            )
                final_qos_state, _ = self.impl.read_all_state()
                LOG.info("final_qos_state -> %s", json.dumps(final_qos_state, indent=1))
                LOG.info("final_redis state -> %s", self._redis_store)
            except Exception as e:  # pylint: disable=broad-except
                # in case of any exception start clean slate

                LOG.error("error %s. restarting clean %s", e, traceback.format_exc())
                self._clean_restart = True

            self._initialized = True

    def _get_or_create_subscriber(self, imsi):
        subscriber_state = self._subscriber_state.get(imsi)
        if not subscriber_state:
            subscriber_state = SubscriberState(imsi, self._redis_store)
            self._subscriber_state[imsi] = subscriber_state
        return subscriber_state

    def add_subscriber_qos(
            self,
            imsi: str,
            ip_addr: str,
            apn_ambr: int,
            units: str,
            rule_num: int,
            direction: FlowMatch.Direction,
            qos_info: QosInfo,
            cleanup_rule=None,
    ):
        with self._lock:
            if not self._qos_enabled or not self._initialized:
                LOG.debug("add_subscriber_qos: not enabled or initialized")
                return None, None, None

            LOG.debug(
                "adding qos for imsi %s rule_num %d direction %d apn_ambr %d, %s",
                imsi, rule_num, direction, apn_ambr, qos_info,
            )

            imsi = normalize_imsi(imsi)

            # ip_addr identifies a specific subscriber session, each subscriber session
            # must be associated with a default bearer and can be associated with dedicated
            # bearers. APN AMBR specifies the aggregate max bit rate for a specific
            # subscriber across all the bearers. Queues for dedicated bearers will be
            # children of default bearer Queues. In case the dedicated bearers exceed the
            # rate, then they borrow from the default bearer queue
            subscriber_state = self._get_or_create_subscriber(imsi)

            qos_handle = subscriber_state.get_qos_handle(rule_num, direction)
            if qos_handle:
                LOG.debug(
                    "qos exists for imsi %s rule_num %d direction %d",
                    imsi, rule_num, direction,
                )

                return self.impl.get_action_instruction(qos_handle)

            ambr_qos_handle_root = 0
            ambr_qos_handle_leaf = 0
            if self._apn_ambr_enabled and apn_ambr > 0:
                session = subscriber_state.get_or_create_session(ip_addr)
                ambr_qos_handle_root = session.get_ambr(direction)
                LOG.debug("existing root rec: ambr_qos_handle_root %d", ambr_qos_handle_root)

                if not ambr_qos_handle_root:
                    ambr_qos_handle_root = self.impl.add_qos(
                        direction, QosInfo(gbr=None, mbr=apn_ambr),
                        cleanup_rule, units=units,
                        skip_filter=True,
                    )
                    if not ambr_qos_handle_root:
                        LOG.error(
                            'Failed adding root ambr qos mbr %u direction %d',
                            apn_ambr, direction,
                        )
                        return None, None, None
                    else:
                        LOG.debug(
                            'Added root ambr qos mbr %u direction %d qos_handle %d ',
                            apn_ambr, direction, ambr_qos_handle_root,
                        )

                ambr_qos_handle_leaf = session.get_ambr_leaf(direction)
                LOG.debug("existing leaf rec: ambr_qos_handle_leaf %d", ambr_qos_handle_leaf)

                if not ambr_qos_handle_leaf:
                    ambr_qos_handle_leaf = self.impl.add_qos(
                        direction,
                        QosInfo(gbr=None, mbr=apn_ambr),
                        cleanup_rule, units=units,
                        parent=ambr_qos_handle_root,
                    )
                    if ambr_qos_handle_leaf:
                        session.set_ambr(direction, ambr_qos_handle_root, ambr_qos_handle_leaf)
                        LOG.debug(
                            'Added ambr qos mbr %u direction %d qos_handle %d/%d ',
                            apn_ambr, direction, ambr_qos_handle_root, ambr_qos_handle_leaf,
                        )
                    else:
                        LOG.error(
                            'Failed adding leaf ambr qos mbr %u direction %d',
                            apn_ambr, direction,
                        )
                        self.impl.remove_qos(ambr_qos_handle_root, direction, skip_filter=True)
                        return None, None, None
                qos_handle = ambr_qos_handle_leaf

            if qos_info:
                qos_handle = self.impl.add_qos(
                    direction, qos_info, cleanup_rule, units=units,
                    parent=ambr_qos_handle_root,
                )
                LOG.debug("Added ded brr handle: %d", qos_handle)
                if qos_handle:
                    LOG.debug(
                        'Adding qos %s direction %d qos_handle %d ',
                        qos_info, direction, qos_handle,
                    )
                else:
                    LOG.error('Failed adding qos %s direction %d', qos_info, direction)
                    return None, None, None

            if qos_handle:
                subscriber_state.update_rule(
                    ip_addr, rule_num, direction,
                    qos_handle, ambr_qos_handle_root, ambr_qos_handle_leaf,
                )
                return self.impl.get_action_instruction(qos_handle)
            return None, None, None

    def remove_subscriber_qos(self, imsi: str = "", del_rule_num: int = -1):
        with self._lock:
            if not self._qos_enabled or not self._initialized:
                LOG.debug("remove_subscriber_qos: not enabled or initialized")
                return

            LOG.debug("removing Qos for imsi %s del_rule_num %d", imsi, del_rule_num)
            if not imsi:
                LOG.error('imsi %s invalid, failed removing', imsi)
                return

            imsi = normalize_imsi(imsi)
            subscriber_state = self._subscriber_state.get(imsi)
            if not subscriber_state:
                LOG.debug('imsi %s not found, nothing to remove ', imsi)
                return

            to_be_deleted_rules = set()
            qid_to_remove = {}
            qid_in_use = set()
            if del_rule_num == -1:
                # deleting all rules for the subscriber
                rules = subscriber_state.get_all_rules()
                for (rule_num, rule) in rules.items():
                    for (d, qos_data) in rule:
                        _, qid, ambr, leaf = get_data(qos_data)
                        if ambr != qid:
                            qid_to_remove[qid] = d
                            if leaf and leaf != qid:
                                qid_to_remove[leaf] = d

                    to_be_deleted_rules.add(rule_num)
            else:
                rule = subscriber_state.find_rule(del_rule_num)
                if rule:
                    rules = subscriber_state.get_all_rules()
                    for (rule_num, rule) in rules.items():
                        for (d, qos_data) in rule:
                            _, qid, ambr, leaf = get_data(qos_data)
                            if rule_num == del_rule_num:
                                if ambr != qid:
                                    qid_to_remove[qid] = d
                                    if leaf and leaf != qid:
                                        qid_to_remove[leaf] = d
                            else:
                                qid_in_use.add(qid)

                    LOG.debug("removing rule %s %s ", imsi, del_rule_num)
                    to_be_deleted_rules.add(del_rule_num)
                else:
                    LOG.debug("unable to find rule_num %d for imsi %s", del_rule_num, imsi)

            for (qid, d) in qid_to_remove.items():
                if qid not in qid_in_use:
                    self.impl.remove_qos(qid, d)

            for rule_num in to_be_deleted_rules:
                subscriber_state.remove_rule(rule_num)

            # purge sessions with no rules
            for session in subscriber_state.get_all_empty_sessions():
                for d in (FlowMatch.UPLINK, FlowMatch.DOWNLINK):
                    ambr_qos_handle = session.get_ambr(d)
                    if ambr_qos_handle:
                        LOG.debug("removing root ambr qos handle %d direction %d", ambr_qos_handle, d)
                        self.impl.remove_qos(ambr_qos_handle, d, skip_filter=True)
                LOG.debug("purging session %s %s ", imsi, session.ip_addr)
                subscriber_state.remove_session(session.ip_addr)

            # purge subscriber state with no rules
            if subscriber_state.check_empty():
                LOG.debug("purging subscriber state for %s, empty rules and sessions", imsi)
                del self._subscriber_state[imsi]


# -*- coding: utf-8 -*-
#!/usr/bin/env python3.5

from django.shortcuts import render
from django.http import HttpResponseRedirect
from .forms import RegPeopleForm, RegUserForm

def createuser(request):
    if request.method == "POST":
        uform = RegUserForm(data=request.POST)
        pform = RegPeopleForm(data=request.POST)
        if uform.is_valid() and pform.is_valid():
            user = uform.save()
            people = pform.save(commit=False)
            people = user
            people.save()
        return HttpResponseRedirect('/')
    else:
        uform=RegUserForm()
        pform=RegPeopleForm()
        return render(request, 'registration/registration.html', {'uform': uform, 'pform': pform})


#!/usr/bin/env/python
# coding: utf-8

#  YouTube Data API Collector
#  Mat Morrison @mediaczar
#  Last updated: 2020-03-31

''' Query the YouTube Data API for an individual channel URL or a file list of URLs.
You may also specify which 'page' you'd like to start on (useful when the script
breaks during a long data capture.)

You _must_ supply a URL or file handle at the command line.

Data collected:
* publication date
* title
* description
* duration (ISO8601 duration format - parsed with [isodate](https://pypi.python.org/pypi/isodate))
* view count
* comment count
* likes & dislikes

To run it, you'll need:

1. a very basic knowledge of Python

2. some ability to install Python libraries (for legacy reasons,
I'm using a somewhat non-standard library called
[scraperwiki](https://pypi.python.org/pypi/scraperwiki) to save the data.)

3. (to read the output) some basic knowledge of SQL and
[SQLite](https://www.sqlite.org/) (it comes as standard on OS X,
and I use the excellent [Base 2](https://menial.co.uk/base/) to manage and
query the files that this script produces.)

4. an API Key from the Google Developers site
(get it here: [Google Developer Console](https://console.developers.google.com/)) -
add to SET UP. Take care not to share this publicly.
'''

import requests
import json
import scraperwiki
import isodate
import sys
import argparse

def get_file_contents(filename):
    """ Given a filename,
        return the contents of that file
    """
    try:
        with open(filename, 'r') as f:
            # It's assumed our file contains a single line,
            # with our API key
            return f.read().strip()
    except FileNotFoundError:
        print("'%s' file not found" % filename)

def load_data(request):
    x = json.loads(requests.get(request).text)
    return x

def get_author(url):
    elements = url.split("/")
    elements = [i for i in elements if i] # remove 'None' values

    if len(elements) > 3:
        author = elements[3]
        print(author)

        if elements[2] == "channel":
            authortype = "id"
        elif elements[2] == "user":
            authortype = "forUsername"
        else:
            sys.exit('malformed URL: %s' % url)

    else:
        author = elements[2]
        authortype = "channel"

    return author, authortype

def build_channel_request(author, authortype):
    part = 'snippet,contentDetails,statistics,id'
    field_items = ['snippet(title, publishedAt, description)',
                   'contentDetails/relatedPlaylists/uploads',
                   'statistics(subscriberCount,videoCount,viewCount)',
                   'id']
    x = ('https://www.googleapis.com/youtube/v3/' +
         'channels?part=' + part +
         '&fields=items(' + ','.join(field_items) + ')&' +
         authortype + '=' + author +
         '&key=' + API_key)
    return x

def write_channel(json):
    channel = {}
    channel['id'] = json['items'][0]['id']
    channel['title'] = title
    channel['uploads'] = uploadsId
    channel['subscriberCount'] = json['items'][0]['statistics']['subscriberCount']
    channel['videoCount'] = json['items'][0]['statistics']['videoCount']
    channel['publishedAt'] = json['items'][0]['snippet']['publishedAt']
    channel['description'] = json['items'][0]['snippet']['description']

    scraperwiki.sqlite.save(unique_keys=['id'], table_name='channel', data=channel)

def initialise_playlist(uploadsId):
    part = 'snippet,contentDetails'
    field_items = ['snippet(title, publishedAt, description)',
                   'contentDetails(videoId, duration)',
                   'nextPageToken']
    max_results = '50'

    x = ('https://www.googleapis.com/youtube/v3/' +
         'playlistItems?playlistId=' + uploadsId +
         '&part=' + part +
         #  '&fields=items(' + ','.join(field_items) + ')' +
         '&maxResults=' + max_results +
         '&key=' + API_key)
    return x

    print('getting channel data for: %s' % url)  # log progress

def gather_channel(url):
    global title
    global uploadsId
    author, authortype = get_author(url)
    API_request = build_channel_request(author, authortype)
    channel_data = load_data(API_request)
#   verify data in case bad url
    try:
        title = channel_data['items'][0]['snippet']['title']
    except:
        print("bad URL")
        return
#   need following to get playlistId
    uploadsId = channel_data['items'][0]['contentDetails']['relatedPlaylists']['uploads']
    write_channel(channel_data)

    print('...complete')  # log progress

    # now create a list of the videos for that channel
    print('collecting uploads for: %s (playlist ID: %s)' % (title, uploadsId))     # log progress
    API_request_base = initialise_playlist(uploadsId)

    # test to see if a page has been submitted as a command line option
    if args.page:
        page_token = args.page[0]
    else:
        page_token = ''

    # loop through the playlist collecting video data
    gather_video_data(API_request_base, page_token)

def gather_video_data(API_request_base, page_token):
    global videodata
    while True:
        paging_request = API_request_base + '&pageToken=' + page_token

        if page_token == '':
            print('gathering first page of %s' % title)
        else:
            print('gathering %s page %s' % (title, page_token))

        videos = load_data(paging_request)
        vcount = 1
        videodata = {}
        videodata['channel'] = title

        for video in videos['items']:
            write_video(video)

            if vcount > 1:
                # log progress
                print('\033[1A...completed ' + str(vcount))  # cursor up
            else:
                print('...completed ' + str(vcount))
            vcount += 1

        try:
            # log progress
            print('...gathering next page of videos')
            page_token = videos['nextPageToken']
        except:
            print('...last page reached')
            page_token = ''
            break

def write_video(json):
    videodata['videoId'] = json['contentDetails']['videoId']
    videodata['title'] = json['snippet']['title']
    videodata['publishedAt'] = json['snippet']['publishedAt']
    videodata['description'] = json['snippet']['description']

    video_request = ('https://www.googleapis.com/youtube/v3/' +
                     'videos?part=statistics,contentDetails&id=' +
                     videodata['videoId'] +
                     '&key=' + API_key)

    stats_json = load_data(video_request)
    for stat in ['viewCount', 'likeCount', 'dislikeCount', 'commentCount']:
        try:
            videodata[stat] = int(stats_json['items'][0]['statistics'][stat])
        except:
            videodata[stat] = None

    duration = isodate.parse_duration(stats_json['items'][0]['contentDetails']['duration']).total_seconds()
    videodata['duration'] = duration
    scraperwiki.sqlite.save(unique_keys=['videoId'], table_name='videos', data=videodata)

# SET UP
# credentials
API_key = get_file_contents("api_key")  # obtain from Google Developer Console, store in api_key locally

# parse arguments at command line

parser = argparse.ArgumentParser(description="""Query the YouTube Data API for an individual channel URL or a file list of URLs.
You may also specify which 'page' you'd like to start on (useful when the script breaks during a long data capture.)
You *must* supply a URL or file handle at the command line.""")

group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-u", "--url", nargs=1, type=str, help="url of YouTube channel")
group.add_argument("-f", "--file", nargs=1, type=str, help="file list of URLs")

parser.add_argument("-p", "--page", nargs=1, type=str, help="page code")

args = parser.parse_args()

if args.file:
    f = open(args.file[0])
    urls = [url.rstrip('\n') for url in f]
    f.close
elif args.url:
    urls = args.url

# iterate through the URLs collecting basic channel data
for url in urls:
    print("gathering %s" % url)
    gather_channel(url)


"""
Honk Settings.
"""

import environ
from pathlib import Path

from google.oauth2 import service_account

env = environ.Env(
    # set casting, default value
    DEBUG=(bool, False),
)

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR: Path = Path(__file__).resolve().parent.parent

# reading .env files
environ.Env.read_env(BASE_DIR / '.env')

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY: str = env('SECRET_KEY')

DEBUG: bool = env('DEBUG')

ALLOWED_HOSTS: list[str] = ['localhost', '127.0.0.1', 'honk.rafaelmc.net']

CSRF_TRUSTED_ORIGINS: list[str] = ['https://honk.rafaelmc.net']

# Application definition

INSTALLED_APPS: list[str] = [
    'circus.apps.CircusConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_extensions',
    'rest_framework',
    'rest_framework.authtoken',
]

MIDDLEWARE: list[str] = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'honk.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [BASE_DIR / "templates"],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'honk.wsgi.application'

# Database
# https://docs.djangoproject.com/en/4.2/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': BASE_DIR / 'sqlite_data' / 'db.sqlite3',
    }
}

# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators

VALIDATOR_PATH = 'django.contrib.auth.password_validation.'

AUTH_PASSWORD_VALIDATORS = [
    {'NAME': VALIDATOR_PATH + 'UserAttributeSimilarityValidator'},
    {'NAME': VALIDATOR_PATH + 'MinimumLengthValidator'},
    {'NAME': VALIDATOR_PATH + 'CommonPasswordValidator'},
    {'NAME': VALIDATOR_PATH + 'NumericPasswordValidator'},
]

# Internationalization
# https://docs.djangoproject.com/en/4.2/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.2/howto/static-files/

STATIC_URL = 'static/'

STATICFILES_DIRS = [
    BASE_DIR / "static",
]

# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
LOGIN_REDIRECT_URL = "/"
STORAGES = {
    "default": {"BACKEND": "storages.backends.gcloud.GoogleCloudStorage"},
    "staticfiles": {
        "BACKEND": "storages.backends.gcloud.GoogleCloudStorage"
    },
}

# GOOGLE_APPLICATION_CREDENTIALS =
GS_BUCKET_NAME = 'honkhonk'
MEDIA_URL = "/media/"
MEDIA_ROOT = BASE_DIR / "media"
GS_CREDENTIALS = service_account.Credentials.from_service_account_file(
    f"{BASE_DIR}/gcp-honk-credentials.json"
)

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': [
        'rest_framework.authentication.TokenAuthentication',
    ],
}


import pygame
import serial
import time
import sys

class GUI:
    def __init__(self):
        # screen
        pygame.init()
        self.screen = pygame.display.set_mode((350, 400))
        self.DELTA = 40
        self.wide_value = 7500
        self.angle_y = 3890
        self.angle_x = 1230
        self.ser = serial.Serial('COM6', baudrate=9600, timeout=0.01)
        time.sleep(1)

        # creating buttons`
        button_up_surface = pygame.transform.scale(pygame.image.load("up.jpg"), (50, 50))
        button_down_surface = pygame.transform.scale(pygame.image.load("down.jpg"), (50, 50))
        button_right_surface = pygame.transform.scale(pygame.image.load("right.png"), (50, 50))
        button_left_surface = pygame.transform.scale(pygame.image.load("left.jpg"), (50, 50))
        button_wide_surface = pygame.transform.scale(pygame.image.load("wide.jpg"), (100, 50))
        button_narrow_surface = pygame.transform.scale(pygame.image.load("narrow.png"), (100, 50))
        self.button_up = Button(button_up_surface, 175, 50)
        self.button_down = Button(button_down_surface, 175, 150)
        self.button_right = Button(button_right_surface, 250, 100)
        self.button_left = Button(button_left_surface, 100, 100)
        self.button_wide = Button(button_wide_surface, 175, 225)
        self.button_narrow = Button(button_narrow_surface, 175, 280)

    def listen(self):
        while True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()
                if event.type == pygame.MOUSEBUTTONDOWN:
                    if self.button_down.checking_input(pygame.mouse.get_pos()):
                        self.angle_y -= self.DELTA if self.angle_y >= 3880 else 0
                        self.ser.write((str(self.angle_y) + '\n').encode('utf-8'))
                    elif self.button_up.checking_input(pygame.mouse.get_pos()):
                        self.angle_y += self.DELTA if self.angle_y <= 4110 else 0
                        self.ser.write((str(self.angle_y) + '\n').encode('utf-8'))
                    elif self.button_right.checking_input(pygame.mouse.get_pos()):
                        self.angle_x -= self.DELTA if self.angle_x >= 1230 else 0
                        self.ser.write((str(self.angle_x) + '\n').encode('utf-8'))
                    elif self.button_left.checking_input(pygame.mouse.get_pos()):
                        self.angle_x += self.DELTA if self.angle_x <= 2230 else 0
                        self.ser.write((str(self.angle_x) + '\n').encode('utf-8'))
                    elif self.button_wide.checking_input(pygame.mouse.get_pos()):
                        # wider, from 1500 to 1750
                        self.ser.write((str(7750) + '\n').encode('utf-8'))
                        time.sleep(0.2)
                        self.ser.write((str(7500) + '\n').encode('utf-8'))
                    elif self.button_narrow.checking_input(pygame.mouse.get_pos()):
                        # wider, from 1280 to 1500
                        self.ser.write((str(7300) + '\n').encode('utf-8'))
                        time.sleep(0.2)
                        self.ser.write((str(7500) + '\n').encode('utf-8'))
                    self.screen.fill("white")

            # updating buttons
            self.screen.fill("white")
            self.button_up.update(self.screen)
            self.button_down.update(self.screen)
            self.button_right.update(self.screen)
            self.button_left.update(self.screen)
            self.button_wide.update(self.screen)
            self.button_narrow.update(self.screen)
            pygame.display.update()

    def close_port(self):
        self.ser.close()

class Button:
    def __init__(self, image, x_pos, y_pos):
        self.image = image
        self.x_pos = x_pos
        self.y_pos = y_pos
        self.rect = self.image.get_rect(center=(self.x_pos, self.y_pos))

    def update(self, screen):
        screen.blit(self.image, self.rect)

    def checking_input(self, position):
        if position[0] in range(self.rect.left, self.rect.right) and position[1] in range(self.rect.top,
                                                                                self.rect.bottom):
            return True

class Controller:
    def __init__(self):
        # controller
        pygame.init()
        pygame.joystick.init()
        self.controller = pygame.joystick.Joystick(0)
        self.DELTA = 35
        self.angle_y = 3880
        self.angle_x = 1230
        self.controller.init()
        self.ser = serial.Serial('COM6', baudrate=9600, timeout=0.01)
        time.sleep(1)

    def listen(self):
        while True:
            pygame.event.get()
            if self.controller.get_button(1):
                # right
                self.angle_x -= self.DELTA if self.angle_x >= 1230 else 0
                self.ser.write((str(self.angle_x) + '\n').encode('utf-8'))
                time.sleep(0.1)
            elif self.controller.get_button(2):
                # left
                self.angle_x += self.DELTA if self.angle_x <= 2230 else 0
                self.ser.write((str(self.angle_x) + '\n').encode('utf-8'))
                time.sleep(0.1)
            elif self.controller.get_button(3):
                # up
                self.angle_y += self.DELTA if self.angle_y <= 4110 else 0
                self.ser.write((str(self.angle_y) + '\n').encode('utf-8'))
                time.sleep(0.1)
            elif self.controller.get_button(0):
                self.angle_y -= self.DELTA if self.angle_y >= 3880 else 0
                self.ser.write((str(self.angle_y) + '\n').encode('utf-8'))
                time.sleep(0.1)
            elif round(self.controller.get_axis(5), 2) != -1.0:
                # wider
                axis_data = round(self.controller.get_axis(5), 2)
                new_value = int(self.map_data(axis_data, -1.0, 1.0, 7500, 7750))
                self.ser.write((str(new_value) + '\n').encode('utf-8'))
                time.sleep(0.1)
            elif round(self.controller.get_axis(4), 2) != -1.0:
                # narrower
                axis_data = round(self.controller.get_axis(4), 2)
                new_value = int(self.map_data(axis_data, -1.0, 1.0, 7500, 7280))
                self.ser.write((str(new_value) + '\n').encode('utf-8'))
                time.sleep(0.1)

    def close_port(self):
        self.ser.close()

    def map_data(self, x, in_min, in_max, out_min, out_max):
        # converting one interval to another
        return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min

if __name__ == "__main__":
    try:
        try:
            ps4 = Controller()
            ps4.listen()
        except KeyboardInterrupt:
            ps4.close_port()
    except pygame.error:
        try:
            gui = GUI()
            gui.listen()
        except KeyboardInterrupt:
            gui.close_port()