<commit_before>// Copyright 2018 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
package devices

import (
	"fmt"
	"regexp"
)

// Driver names
const (
	DriverHvNetvcs      = "hv_netvcs"
	DriverUioPciGeneric = "uio_pci_generic"
	DriverIgbUio        = "igb_uio"
	DriverVfioPci       = "vfio-pci"
	DriverUioHvGeneric  = "uio_hv_generic"
)

// Path to PCI
const (
	PathSysPciDevices     = "/sys/bus/pci/devices"
	PathSysPciDrivers     = "/sys/bus/pci/drivers"
	PathSysPciDriverProbe = "/sys/bus/pci/drivers_probe"
)

// Path to VMBus
const (
	PathSysVmbusDevices = "/sys/bus/vmbus/devices"
	PathSysVmbusDrivers = "/sys/bus/vmbus/drivers"
)

// Path to net
const (
	PathSysClassNet = "/sys/class/net"
)

// Regular expressions for PCI-ID and UUID
var (
	IsPciID *regexp.Regexp
	IsUUID  *regexp.Regexp
)

// DPDK related drivers
var (
	DefaultDpdkDriver = DriverUioPciGeneric
	DpdkDrivers       = [...]string{DriverUioPciGeneric, DriverIgbUio, DriverVfioPci, DriverUioHvGeneric}
	DpdkPciDrivers    = [...]string{DriverUioPciGeneric, DriverIgbUio, DriverVfioPci}
	DpdkVmbusDrivers  = [...]string{DriverUioHvGeneric}
)

type stringBuilder string

func (s stringBuilder) With(args ...interface{}) string {
	return fmt.Sprintf(string(s), args...)
}

var (
	pathSysPciDevicesBind           stringBuilder = PathSysPciDevices + "/%s/driver/bind"
	pathSysPciDevicesUnbind         stringBuilder = PathSysPciDevices + "/%s/driver/unbind"
	pathSysPciDevicesOverrideDriver stringBuilder = PathSysPciDevices + "/%s/driver_override"

	pathSysPciDriversBind   stringBuilder = PathSysPciDrivers + "/%s/bind"
	pathSysPciDriversUnbind stringBuilder = PathSysPciDrivers + "/%s/unbind"
	pathSysPciDriversNewID  stringBuilder = PathSysPciDrivers + "/%s/new_id"
)

var (
	pathSysVmbusDriversBind   stringBuilder = PathSysVmbusDrivers + "/%s/bind"
	pathSysVmbusDriversUnbind stringBuilder = PathSysVmbusDrivers + "/%s/unbind"
	pathSysVmbusDriversNewID  stringBuilder = PathSysVmbusDrivers + "/%s/new_id"
)

var (
	pathSysClassNetDeviceDriver stringBuilder = PathSysClassNet + "/%s/device/driver"
	pathSysClassNetDevice       stringBuilder = PathSysClassNet + "/%s/device"
)

var (
	vmbusDeviceStringer stringBuilder = "ID: %s\nClass:\t%s\nDriver:\t%s"
	pciDeviceStringer   stringBuilder = "ID: %s\nClass:\t%s\nVendor:\t%s\nDevice:\t%s\nDriver:\t%s"
)

var (
	// for ethtool output
	rPciIDForEthtool *regexp.Regexp

	// for lspci output
	rPciClass  *regexp.Regexp
	rPciVendor *regexp.Regexp
	rPciDevice *regexp.Regexp
	rPciDriver *regexp.Regexp

	// for find output
	rVmbusDriver *regexp.Regexp
)

func init() {
	rPciIDForEthtool = regexp.MustCompile("bus-info:\\s([0-9:.]+)")

	rPciClass = regexp.MustCompile("[Cc]lass:\\s([0-9a-zA-Z]{4})")
	rPciVendor = regexp.MustCompile("[Vv]endor:\\s([0-9a-zA-Z]{4})")
	rPciDevice = regexp.MustCompile("[Dd]evice:\\s([0-9a-zA-Z]{4})")
	rPciDriver = regexp.MustCompile("[Dd]river:\\s(\\S+)")

	rVmbusDriver = regexp.MustCompile("/sys/bus/vmbus/drivers/(\\S+)/")

	IsPciID = regexp.MustCompile("^\\d{4}:\\d{2}:\\d{2}.\\d$")
	IsUUID = regexp.MustCompile("^[[:xdigit:]]{8}-[[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$")
}
<commit_msg>fix is pci regular expression<commit_after>// Copyright 2018 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
package devices

import (
	"fmt"
	"regexp"
)

// Driver names
const (
	DriverHvNetvcs      = "hv_netvcs"
	DriverUioPciGeneric = "uio_pci_generic"
	DriverIgbUio        = "igb_uio"
	DriverVfioPci       = "vfio-pci"
	DriverUioHvGeneric  = "uio_hv_generic"
)

// Path to PCI
const (
	PathSysPciDevices     = "/sys/bus/pci/devices"
	PathSysPciDrivers     = "/sys/bus/pci/drivers"
	PathSysPciDriverProbe = "/sys/bus/pci/drivers_probe"
)

// Path to VMBus
const (
	PathSysVmbusDevices = "/sys/bus/vmbus/devices"
	PathSysVmbusDrivers = "/sys/bus/vmbus/drivers"
)

// Path to net
const (
	PathSysClassNet = "/sys/class/net"
)

// Regular expressions for PCI-ID and UUID
var (
	IsPciID *regexp.Regexp
	IsUUID  *regexp.Regexp
)

// DPDK related drivers
var (
	DefaultDpdkDriver = DriverUioPciGeneric
	DpdkDrivers       = [...]string{DriverUioPciGeneric, DriverIgbUio, DriverVfioPci, DriverUioHvGeneric}
	DpdkPciDrivers    = [...]string{DriverUioPciGeneric, DriverIgbUio, DriverVfioPci}
	DpdkVmbusDrivers  = [...]string{DriverUioHvGeneric}
)

type stringBuilder string

func (s stringBuilder) With(args ...interface{}) string {
	return fmt.Sprintf(string(s), args...)
}

var (
	pathSysPciDevicesBind           stringBuilder = PathSysPciDevices + "/%s/driver/bind"
	pathSysPciDevicesUnbind         stringBuilder = PathSysPciDevices + "/%s/driver/unbind"
	pathSysPciDevicesOverrideDriver stringBuilder = PathSysPciDevices + "/%s/driver_override"

	pathSysPciDriversBind   stringBuilder = PathSysPciDrivers + "/%s/bind"
	pathSysPciDriversUnbind stringBuilder = PathSysPciDrivers + "/%s/unbind"
	pathSysPciDriversNewID  stringBuilder = PathSysPciDrivers + "/%s/new_id"
)

var (
	pathSysVmbusDriversBind   stringBuilder = PathSysVmbusDrivers + "/%s/bind"
	pathSysVmbusDriversUnbind stringBuilder = PathSysVmbusDrivers + "/%s/unbind"
	pathSysVmbusDriversNewID  stringBuilder = PathSysVmbusDrivers + "/%s/new_id"
)

var (
	pathSysClassNetDeviceDriver stringBuilder = PathSysClassNet + "/%s/device/driver"
	pathSysClassNetDevice       stringBuilder = PathSysClassNet + "/%s/device"
)

var (
	vmbusDeviceStringer stringBuilder = "ID: %s\nClass:\t%s\nDriver:\t%s"
	pciDeviceStringer   stringBuilder = "ID: %s\nClass:\t%s\nVendor:\t%s\nDevice:\t%s\nDriver:\t%s"
)

var (
	// for ethtool output
	rPciIDForEthtool *regexp.Regexp

	// for lspci output
	rPciClass  *regexp.Regexp
	rPciVendor *regexp.Regexp
	rPciDevice *regexp.Regexp
	rPciDriver *regexp.Regexp

	// for find output
	rVmbusDriver *regexp.Regexp
)

func init() {
	rPciIDForEthtool = regexp.MustCompile("bus-info:\\s([0-9:.]+)")

	rPciClass = regexp.MustCompile("[Cc]lass:\\s([0-9a-zA-Z]{4})")
	rPciVendor = regexp.MustCompile("[Vv]endor:\\s([0-9a-zA-Z]{4})")
	rPciDevice = regexp.MustCompile("[Dd]evice:\\s([0-9a-zA-Z]{4})")
	rPciDriver = regexp.MustCompile("[Dd]river:\\s(\\S+)")

	rVmbusDriver = regexp.MustCompile("/sys/bus/vmbus/drivers/(\\S+)/")

	// domains are numbered from 0 to ffff), bus (0 to ff), slot (0 to 1f) and function (0 to 7)
	IsPciID = regexp.MustCompile("^[[:xdigit:]]{4}:[[:xdigit:]]{2}:[0-1][[:xdigit:]].[0-7]$")
	IsUUID = regexp.MustCompile("^[[:xdigit:]]{8}-[[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$")
}
<|endoftext|>


<commit_before>package sdl

// #include "SDL.h"
import "C"

import (
	"unsafe"
)

// WindowFlag is a window creation option.
type WindowFlag uint32

// Window creation options.
const (
	WindowFullscreen        WindowFlag = C.SDL_WINDOW_FULLSCREEN
	WindowOpenGL            WindowFlag = C.SDL_WINDOW_OPENGL
	WindowShown             WindowFlag = C.SDL_WINDOW_SHOWN
	WindowHidden            WindowFlag = C.SDL_WINDOW_HIDDEN
	WindowBorderless        WindowFlag = C.SDL_WINDOW_BORDERLESS
	WindowResizable         WindowFlag = C.SDL_WINDOW_RESIZABLE
	WindowMinimized         WindowFlag = C.SDL_WINDOW_MINIMIZED
	WindowMaximized         WindowFlag = C.SDL_WINDOW_MAXIMIZED
	WindowInputGrabbed      WindowFlag = C.SDL_WINDOW_INPUT_GRABBED
	WindowInputFocus        WindowFlag = C.SDL_WINDOW_INPUT_FOCUS
	WindowMouseFocus        WindowFlag = C.SDL_WINDOW_MOUSE_FOCUS
	WindowForeign           WindowFlag = C.SDL_WINDOW_FOREIGN
	WindowAllowHighDPI      WindowFlag = C.SDL_WINDOW_ALLOW_HIGHDPI
	WindowFullscreenDesktop WindowFlag = C.SDL_WINDOW_FULLSCREEN_DESKTOP
)

// Window positions.  Used for the origin in NewWindow.
const (
	WindowPosUndefined = C.SDL_WINDOWPOS_UNDEFINED
	WindowPosCentered  = C.SDL_WINDOWPOS_CENTERED
)

// RendererInfo describes the capabilities of a Renderer.
type RendererInfo struct {
	Name             string
	Flags            RendererFlag
	TextureFormats   []PixelFormatEnum
	MaxTextureWidth  int
	MaxTextureHeight int
}

// Window is a window in a GUI environment.
type Window struct {
	w C.SDL_Window
}

// NewWindow creates a new window.  r.x or r.y may also be
// WindowPosCentered or WindowPosUndefined.  Multiple flags will be
// ORed together.
func NewWindow(title string, r Rectangle, flags ...WindowFlag) (*Window, error) {
	ctitle := C.CString(title)
	defer C.free(unsafe.Pointer(ctitle))
	var f WindowFlag
	for i := range flags {
		f |= flags[i]
	}
	w := C.SDL_CreateWindow(
		ctitle,
		C.int(r.Origin.X),
		C.int(r.Origin.Y),
		C.int(r.W),
		C.int(r.H),
		C.Uint32(f))
	if w == nil {
		return nil, GetError()
	}
	return (*Window)(unsafe.Pointer(w)), nil
}

// Surface returns the window's surface.
func (w *Window) Surface() *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_GetWindowSurface(&w.w)))
}

// CreateRenderer creates a 2D rendering context for a window.
// driverIndex is the index of the rendering driver to initialize, or -1
// to initialize the first one that supports the requested
// configuration.  Multiple flags will be ORed together.
func (w *Window) CreateRenderer(driverIndex int, flags ...RendererFlag) (*Renderer, error) {
	var f RendererFlag
	for i := range flags {
		f |= flags[i]
	}
	r := C.SDL_CreateRenderer(&w.w, C.int(driverIndex), C.Uint32(f))
	if r == nil {
		return nil, GetError()
	}
	return (*Renderer)(unsafe.Pointer(r)), nil
}

// Renderer returns the window's renderer or nil if it doesn't have one.
func (w *Window) Renderer() *Renderer {
	return (*Renderer)(unsafe.Pointer(C.SDL_GetRenderer(&w.w)))
}

// Destroy destroys a window.  It is not safe to use the window after
// calling Destroy.
func (w *Window) Destroy() {
	C.SDL_DestroyWindow(&w.w)
}

// A Renderer represents the rendering state.
type Renderer struct {
	r C.SDL_Renderer
}

// A RendererFlag is an option for creating a renderer.
type RendererFlag uint32

// Renderer creation flags.
const (
	// Software fallback
	RendererSoftware RendererFlag = C.SDL_RENDERER_SOFTWARE
	// Hardware accelerated
	RendererAccelerated RendererFlag = C.SDL_RENDERER_ACCELERATED
	// Present is synchronized with the refresh rate
	RendererPresentVSync RendererFlag = C.SDL_RENDERER_PRESENTVSYNC
	// Render to texture support
	RendererTargetTexture RendererFlag = C.SDL_RENDERER_TARGETTEXTURE
)

// Info returns the renderer's capabilities.
func (r *Renderer) Info() (*RendererInfo, error) {
	var info C.SDL_RendererInfo
	if C.SDL_GetRendererInfo(&r.r, &info) != 0 {
		return nil, GetError()
	}

	formats := make([]PixelFormatEnum, info.num_texture_formats)
	for i := range formats {
		formats[i] = PixelFormatEnum(info.texture_formats[i])
	}
	return &RendererInfo{
		Name:             C.GoString(info.name),
		Flags:            RendererFlag(info.flags),
		TextureFormats:   formats,
		MaxTextureWidth:  int(info.max_texture_width),
		MaxTextureHeight: int(info.max_texture_height),
	}, nil
}

// CopyTexture copies a portion of the texture to the current rendering context.
func (r *Renderer) CopyTexture(texture *Texture, srcRect, destRect *Rectangle) error {
	if C.SDL_RenderCopy(&r.r, &texture.t, srcRect.toCRect(), destRect.toCRect()) != 0 {
		return GetError()
	}
	return nil
}

// Clear clears the current rendering target with the drawing color.
func (r *Renderer) Clear() error {
	if C.SDL_RenderClear(&r.r) != 0 {
		return GetError()
	}
	return nil
}

// Present updates the screen with rendering performed.
func (r *Renderer) Present() {
	C.SDL_RenderPresent(&r.r)
}

// Destroy destroys the renderer.  The renderer should not be used after
// calling Destroy.
func (r *Renderer) Destroy() {
	C.SDL_DestroyRenderer(&r.r)
}
<commit_msg>Add Size() function to Window (SDL_GetWindowSize)<commit_after>package sdl

// #include "SDL.h"
import "C"

import (
	"unsafe"
)

// WindowFlag is a window creation option.
type WindowFlag uint32

// Window creation options.
const (
	WindowFullscreen        WindowFlag = C.SDL_WINDOW_FULLSCREEN
	WindowOpenGL            WindowFlag = C.SDL_WINDOW_OPENGL
	WindowShown             WindowFlag = C.SDL_WINDOW_SHOWN
	WindowHidden            WindowFlag = C.SDL_WINDOW_HIDDEN
	WindowBorderless        WindowFlag = C.SDL_WINDOW_BORDERLESS
	WindowResizable         WindowFlag = C.SDL_WINDOW_RESIZABLE
	WindowMinimized         WindowFlag = C.SDL_WINDOW_MINIMIZED
	WindowMaximized         WindowFlag = C.SDL_WINDOW_MAXIMIZED
	WindowInputGrabbed      WindowFlag = C.SDL_WINDOW_INPUT_GRABBED
	WindowInputFocus        WindowFlag = C.SDL_WINDOW_INPUT_FOCUS
	WindowMouseFocus        WindowFlag = C.SDL_WINDOW_MOUSE_FOCUS
	WindowForeign           WindowFlag = C.SDL_WINDOW_FOREIGN
	WindowAllowHighDPI      WindowFlag = C.SDL_WINDOW_ALLOW_HIGHDPI
	WindowFullscreenDesktop WindowFlag = C.SDL_WINDOW_FULLSCREEN_DESKTOP
)

// Window positions.  Used for the origin in NewWindow.
const (
	WindowPosUndefined = C.SDL_WINDOWPOS_UNDEFINED
	WindowPosCentered  = C.SDL_WINDOWPOS_CENTERED
)

// RendererInfo describes the capabilities of a Renderer.
type RendererInfo struct {
	Name             string
	Flags            RendererFlag
	TextureFormats   []PixelFormatEnum
	MaxTextureWidth  int
	MaxTextureHeight int
}

// Window is a window in a GUI environment.
type Window struct {
	w C.SDL_Window
}

// NewWindow creates a new window.  r.x or r.y may also be
// WindowPosCentered or WindowPosUndefined.  Multiple flags will be
// ORed together.
func NewWindow(title string, r Rectangle, flags ...WindowFlag) (*Window, error) {
	ctitle := C.CString(title)
	defer C.free(unsafe.Pointer(ctitle))
	var f WindowFlag
	for i := range flags {
		f |= flags[i]
	}
	w := C.SDL_CreateWindow(
		ctitle,
		C.int(r.Origin.X),
		C.int(r.Origin.Y),
		C.int(r.W),
		C.int(r.H),
		C.Uint32(f))
	if w == nil {
		return nil, GetError()
	}
	return (*Window)(unsafe.Pointer(w)), nil
}

// Surface returns the window's surface.
func (w *Window) Surface() *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_GetWindowSurface(&w.w)))
}

// CreateRenderer creates a 2D rendering context for a window.
// driverIndex is the index of the rendering driver to initialize, or -1
// to initialize the first one that supports the requested
// configuration.  Multiple flags will be ORed together.
func (w *Window) CreateRenderer(driverIndex int, flags ...RendererFlag) (*Renderer, error) {
	var f RendererFlag
	for i := range flags {
		f |= flags[i]
	}
	r := C.SDL_CreateRenderer(&w.w, C.int(driverIndex), C.Uint32(f))
	if r == nil {
		return nil, GetError()
	}
	return (*Renderer)(unsafe.Pointer(r)), nil
}

// Renderer returns the window's renderer or nil if it doesn't have one.
func (w *Window) Renderer() *Renderer {
	return (*Renderer)(unsafe.Pointer(C.SDL_GetRenderer(&w.w)))
}

// Size returns the size of the SDL window, in pixels.
func (w *Window) Size() (int, int) {
	var width C.int
	var height C.int
	C.SDL_GetWindowSize(&w.w, &width, &height)
	return int(width), int(height)
}

// Destroy destroys a window.  It is not safe to use the window after
// calling Destroy.
func (w *Window) Destroy() {
	C.SDL_DestroyWindow(&w.w)
}

// A Renderer represents the rendering state.
type Renderer struct {
	r C.SDL_Renderer
}

// A RendererFlag is an option for creating a renderer.
type RendererFlag uint32

// Renderer creation flags.
const (
	// Software fallback
	RendererSoftware RendererFlag = C.SDL_RENDERER_SOFTWARE
	// Hardware accelerated
	RendererAccelerated RendererFlag = C.SDL_RENDERER_ACCELERATED
	// Present is synchronized with the refresh rate
	RendererPresentVSync RendererFlag = C.SDL_RENDERER_PRESENTVSYNC
	// Render to texture support
	RendererTargetTexture RendererFlag = C.SDL_RENDERER_TARGETTEXTURE
)

// Info returns the renderer's capabilities.
func (r *Renderer) Info() (*RendererInfo, error) {
	var info C.SDL_RendererInfo
	if C.SDL_GetRendererInfo(&r.r, &info) != 0 {
		return nil, GetError()
	}

	formats := make([]PixelFormatEnum, info.num_texture_formats)
	for i := range formats {
		formats[i] = PixelFormatEnum(info.texture_formats[i])
	}
	return &RendererInfo{
		Name:             C.GoString(info.name),
		Flags:            RendererFlag(info.flags),
		TextureFormats:   formats,
		MaxTextureWidth:  int(info.max_texture_width),
		MaxTextureHeight: int(info.max_texture_height),
	}, nil
}

// CopyTexture copies a portion of the texture to the current rendering context.
func (r *Renderer) CopyTexture(texture *Texture, srcRect, destRect *Rectangle) error {
	if C.SDL_RenderCopy(&r.r, &texture.t, srcRect.toCRect(), destRect.toCRect()) != 0 {
		return GetError()
	}
	return nil
}

// Clear clears the current rendering target with the drawing color.
func (r *Renderer) Clear() error {
	if C.SDL_RenderClear(&r.r) != 0 {
		return GetError()
	}
	return nil
}

// Present updates the screen with rendering performed.
func (r *Renderer) Present() {
	C.SDL_RenderPresent(&r.r)
}

// Destroy destroys the renderer.  The renderer should not be used after
// calling Destroy.
func (r *Renderer) Destroy() {
	C.SDL_DestroyRenderer(&r.r)
}
<|endoftext|>


<commit_before>package mastodon

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

type Config struct {
	Server       string
	ClientID     string
	ClientSecret string
	AccessToken  string
}

type client struct {
	http.Client
	config *Config
}

func (c *client) doAPI(method string, uri string, params url.Values, res interface{}) error {
	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, uri)

	var resp *http.Response
	req, err := http.NewRequest(method, url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
	resp, err = c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if res == nil {
		return nil
	}
	return json.NewDecoder(resp.Body).Decode(&res)
}

func NewClient(config *Config) *client {
	return &client{
		Client: *http.DefaultClient,
		config: config,
	}
}

func (c *client) Authenticate(username, password string) error {
	params := url.Values{}
	params.Set("client_id", c.config.ClientID)
	params.Set("client_secret", c.config.ClientSecret)
	params.Set("grant_type", "password")
	params.Set("username", username)
	params.Set("password", password)
	params.Set("scope", "read write follow")

	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, "/oauth/token")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	res := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return err
	}
	c.config.AccessToken = res.AccessToken
	return nil
}

// AppConfig is a setting for registering applications.
type AppConfig struct {
	http.Client
	Server     string
	ClientName string

	// Where the user should be redirected after authorization (for no redirect, use urn:ietf:wg:oauth:2.0:oob)
	RedirectURIs string

	// This can be a space-separated list of the following items: "read", "write" and "follow".
	Scopes string

	// Optional.
	Website string
}

// Application is mastodon application.
type Application struct {
	ID           int64  `json:"id"`
	RedirectURI  string `json:"redirect_uri"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

// RegisterApp returns the mastodon application.
func RegisterApp(appConfig *AppConfig) (*Application, error) {
	params := url.Values{}
	params.Set("client_name", appConfig.ClientName)
	params.Set("redirect_uris", appConfig.RedirectURIs)
	params.Set("scopes", appConfig.Scopes)
	params.Set("website", appConfig.Website)

	url, err := url.Parse(appConfig.Server)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, "/api/v1/apps")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return nil, err
	}
	resp, err := appConfig.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	app := &Application{}
	err = json.NewDecoder(resp.Body).Decode(app)
	if err != nil {
		return nil, err
	}

	return app, nil
}

type Account struct {
	ID             int64     `json:"id"`
	Username       string    `json:"username"`
	Acct           string    `json:"acct"`
	DisplayName    string    `json:"display_name"`
	Locked         bool      `json:"locked"`
	CreatedAt      time.Time `json:"created_at"`
	FollowersCount int64     `json:"followers_count"`
	FollowingCount int64     `json:"following_count"`
	StatusesCount  int64     `json:"statuses_count"`
	Note           string    `json:"note"`
	URL            string    `json:"url"`
	Avatar         string    `json:"avatar"`
	AvatarStatic   string    `json:"avatar_static"`
	Header         string    `json:"header"`
	HeaderStatic   string    `json:"header_static"`
}

type Visibility int64

type Toot struct {
	Status      string  `json:"status"`
	InReplyToID int64   `json:"in_reply_to_id"`
	MediaIDs    []int64 `json:"in_reply_to_id"`
	Sensitive   bool    `json:"sensitive"`
	SpoilerText string  `json:"spoiler_text"`
	Visibility  string  `json:"visibility"`
}

type Status struct {
	ID                 int64         `json:"id"`
	CreatedAt          time.Time     `json:"created_at"`
	InReplyToID        interface{}   `json:"in_reply_to_id"`
	InReplyToAccountID interface{}   `json:"in_reply_to_account_id"`
	Sensitive          bool          `json:"sensitive"`
	SpoilerText        string        `json:"spoiler_text"`
	Visibility         string        `json:"visibility"`
	Application        interface{}   `json:"application"`
	Account            Account       `json:"account"`
	MediaAttachments   []interface{} `json:"media_attachments"`
	Mentions           []interface{} `json:"mentions"`
	Tags               []interface{} `json:"tags"`
	URI                string        `json:"uri"`
	Content            string        `json:"content"`
	URL                string        `json:"url"`
	ReblogsCount       int64         `json:"reblogs_count"`
	FavouritesCount    int64         `json:"favourites_count"`
	Reblog             interface{}   `json:"reblog"`
	Favourited         interface{}   `json:"favourited"`
	Reblogged          interface{}   `json:"reblogged"`
}

func (c *client) GetAccount(id int) (*Account, error) {
	var account Account
	err := c.doAPI("GET", fmt.Sprintf("/api/v1/accounts/%d", id), nil, &account)
	if err != nil {
		return nil, err
	}
	return &account, nil
}

func (c *client) GetTimelineHome() ([]*Status, error) {
	var statuses []*Status
	err := c.doAPI("GET", "/api/v1/timelines/home", nil, &statuses)
	if err != nil {
		return nil, err
	}
	return statuses, nil
}

func (c *client) PostStatus(toot *Toot) (*Status, error) {
	params := url.Values{}
	params.Set("status", toot.Status)
	if toot.InReplyToID > 0 {
		params.Set("in_reply_to_id", fmt.Sprint(toot.InReplyToID))
	}
	// TODO: media_ids, senstitive, spoiler_text, visibility
	//params.Set("visibility", "public")

	var status Status
	err := c.doAPI("POST", "/api/v1/statuses", params, &status)
	if err != nil {
		return nil, err
	}
	return &status, nil
}

type UpdateEvent struct {
	Status *Status
}

func (e *UpdateEvent) event() {}

type NotificationEvent struct {
}

func (e *NotificationEvent) event() {}

type DeleteEvent struct {
	ID int64
}

func (e *DeleteEvent) event() {}

type ErrorEvent struct {
	err error
}

func (e *ErrorEvent) Error() string { return e.err.Error() }
func (e *ErrorEvent) event()        {}

type Event interface {
	event()
}

func (c *client) StreamingPublic(ctx context.Context) (chan Event, error) {
	url, err := url.Parse(c.config.Server)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, "/api/v1/streaming/public")

	var resp *http.Response

	q := make(chan Event, 10)
	go func() {
		defer ctx.Done()

		for {
			req, err := http.NewRequest("GET", url.String(), nil)
			if err == nil {
				req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
				resp, err = c.Do(req)
			}
			if err == nil {
				name := ""
				s := bufio.NewScanner(io.TeeReader(resp.Body, os.Stdout))
				for s.Scan() {
					line := s.Text()
					token := strings.SplitN(line, ":", 2)
					if len(token) != 2 {
						continue
					}
					switch strings.TrimSpace(token[0]) {
					case "event":
						name = strings.TrimSpace(token[1])
					case "data":
						switch name {
						case "update":
							var status Status
							json.Unmarshal([]byte(token[1]), &status)
							q <- &UpdateEvent{&status}
						case "notification":
						case "delete":
						}
					default:
					}
				}
				resp.Body.Close()
				err = ctx.Err()
				if err == nil {
					break
				}
			} else {
				q <- &ErrorEvent{err}
			}
			time.Sleep(3 * time.Second)
		}
	}()
	go func() {
		<-ctx.Done()
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()
	return q, nil
}
<commit_msg>golint & go vet<commit_after>package mastodon

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

// Config is a setting for access mastodon APIs.
type Config struct {
	Server       string
	ClientID     string
	ClientSecret string
	AccessToken  string
}

// Client is a API client for mastodon.
type Client struct {
	http.Client
	config *Config
}

func (c *Client) doAPI(method string, uri string, params url.Values, res interface{}) error {
	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, uri)

	var resp *http.Response
	req, err := http.NewRequest(method, url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
	resp, err = c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if res == nil {
		return nil
	}
	return json.NewDecoder(resp.Body).Decode(&res)
}

// NewClient return new mastodon API client.
func NewClient(config *Config) *Client {
	return &Client{
		Client: *http.DefaultClient,
		config: config,
	}
}

// Authenticate get access-token to the API.
func (c *Client) Authenticate(username, password string) error {
	params := url.Values{}
	params.Set("client_id", c.config.ClientID)
	params.Set("client_secret", c.config.ClientSecret)
	params.Set("grant_type", "password")
	params.Set("username", username)
	params.Set("password", password)
	params.Set("scope", "read write follow")

	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, "/oauth/token")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	res := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return err
	}
	c.config.AccessToken = res.AccessToken
	return nil
}

// AppConfig is a setting for registering applications.
type AppConfig struct {
	http.Client
	Server     string
	ClientName string

	// Where the user should be redirected after authorization (for no redirect, use urn:ietf:wg:oauth:2.0:oob)
	RedirectURIs string

	// This can be a space-separated list of the following items: "read", "write" and "follow".
	Scopes string

	// Optional.
	Website string
}

// Application is mastodon application.
type Application struct {
	ID           int64  `json:"id"`
	RedirectURI  string `json:"redirect_uri"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

// RegisterApp returns the mastodon application.
func RegisterApp(appConfig *AppConfig) (*Application, error) {
	params := url.Values{}
	params.Set("client_name", appConfig.ClientName)
	params.Set("redirect_uris", appConfig.RedirectURIs)
	params.Set("scopes", appConfig.Scopes)
	params.Set("website", appConfig.Website)

	url, err := url.Parse(appConfig.Server)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, "/api/v1/apps")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return nil, err
	}
	resp, err := appConfig.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	app := &Application{}
	err = json.NewDecoder(resp.Body).Decode(app)
	if err != nil {
		return nil, err
	}

	return app, nil
}

// Account hold information for mastodon account.
type Account struct {
	ID             int64     `json:"id"`
	Username       string    `json:"username"`
	Acct           string    `json:"acct"`
	DisplayName    string    `json:"display_name"`
	Locked         bool      `json:"locked"`
	CreatedAt      time.Time `json:"created_at"`
	FollowersCount int64     `json:"followers_count"`
	FollowingCount int64     `json:"following_count"`
	StatusesCount  int64     `json:"statuses_count"`
	Note           string    `json:"note"`
	URL            string    `json:"url"`
	Avatar         string    `json:"avatar"`
	AvatarStatic   string    `json:"avatar_static"`
	Header         string    `json:"header"`
	HeaderStatic   string    `json:"header_static"`
}

// Toot is struct to post status.
type Toot struct {
	Status      string  `json:"status"`
	InReplyToID int64   `json:"in_reply_to_id"`
	MediaIDs    []int64 `json:"media_ids"`
	Sensitive   bool    `json:"sensitive"`
	SpoilerText string  `json:"spoiler_text"`
	Visibility  string  `json:"visibility"`
}

// Status is struct to hold status.
type Status struct {
	ID                 int64         `json:"id"`
	CreatedAt          time.Time     `json:"created_at"`
	InReplyToID        interface{}   `json:"in_reply_to_id"`
	InReplyToAccountID interface{}   `json:"in_reply_to_account_id"`
	Sensitive          bool          `json:"sensitive"`
	SpoilerText        string        `json:"spoiler_text"`
	Visibility         string        `json:"visibility"`
	Application        interface{}   `json:"application"`
	Account            Account       `json:"account"`
	MediaAttachments   []interface{} `json:"media_attachments"`
	Mentions           []interface{} `json:"mentions"`
	Tags               []interface{} `json:"tags"`
	URI                string        `json:"uri"`
	Content            string        `json:"content"`
	URL                string        `json:"url"`
	ReblogsCount       int64         `json:"reblogs_count"`
	FavouritesCount    int64         `json:"favourites_count"`
	Reblog             interface{}   `json:"reblog"`
	Favourited         interface{}   `json:"favourited"`
	Reblogged          interface{}   `json:"reblogged"`
}

// GetAccount return Account.
func (c *Client) GetAccount(id int) (*Account, error) {
	var account Account
	err := c.doAPI("GET", fmt.Sprintf("/api/v1/accounts/%d", id), nil, &account)
	if err != nil {
		return nil, err
	}
	return &account, nil
}

// GetTimelineHome return statuses from home timeline.
func (c *Client) GetTimelineHome() ([]*Status, error) {
	var statuses []*Status
	err := c.doAPI("GET", "/api/v1/timelines/home", nil, &statuses)
	if err != nil {
		return nil, err
	}
	return statuses, nil
}

// PostStatus post the toot.
func (c *Client) PostStatus(toot *Toot) (*Status, error) {
	params := url.Values{}
	params.Set("status", toot.Status)
	if toot.InReplyToID > 0 {
		params.Set("in_reply_to_id", fmt.Sprint(toot.InReplyToID))
	}
	// TODO: media_ids, senstitive, spoiler_text, visibility
	//params.Set("visibility", "public")

	var status Status
	err := c.doAPI("POST", "/api/v1/statuses", params, &status)
	if err != nil {
		return nil, err
	}
	return &status, nil
}

// UpdateEvent is struct for passing status event to app.
type UpdateEvent struct {
	Status *Status
}

func (e *UpdateEvent) event() {}

// NotificationEvent is struct for passing notification event to app.
type NotificationEvent struct {
}

func (e *NotificationEvent) event() {}

// DeleteEvent is struct for passing deletion event to app.
type DeleteEvent struct {
	ID int64
}

func (e *DeleteEvent) event() {}

// ErrorEvent is struct for passing errors to app.
type ErrorEvent struct {
	err error
}

func (e *ErrorEvent) Error() string { return e.err.Error() }
func (e *ErrorEvent) event()        {}

// Event is interface passing events to app.
type Event interface {
	event()
}

// StreamingPublic return channel to read events.
func (c *Client) StreamingPublic(ctx context.Context) (chan Event, error) {
	url, err := url.Parse(c.config.Server)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, "/api/v1/streaming/public")

	var resp *http.Response

	q := make(chan Event, 10)
	go func() {
		defer ctx.Done()

		for {
			req, err := http.NewRequest("GET", url.String(), nil)
			if err == nil {
				req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
				resp, err = c.Do(req)
			}
			if err == nil {
				name := ""
				s := bufio.NewScanner(io.TeeReader(resp.Body, os.Stdout))
				for s.Scan() {
					line := s.Text()
					token := strings.SplitN(line, ":", 2)
					if len(token) != 2 {
						continue
					}
					switch strings.TrimSpace(token[0]) {
					case "event":
						name = strings.TrimSpace(token[1])
					case "data":
						switch name {
						case "update":
							var status Status
							json.Unmarshal([]byte(token[1]), &status)
							q <- &UpdateEvent{&status}
						case "notification":
						case "delete":
						}
					default:
					}
				}
				resp.Body.Close()
				err = ctx.Err()
				if err == nil {
					break
				}
			} else {
				q <- &ErrorEvent{err}
			}
			time.Sleep(3 * time.Second)
		}
	}()
	go func() {
		<-ctx.Done()
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()
	return q, nil
}
<|endoftext|>


<commit_before>package sub

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/proto/common"
	"github.com/Symantec/Dominator/sub/scanner"
)

type Configuration struct {
	ScanSpeedPercent    uint
	NetworkSpeedPercent uint
	ScanExclusionList   []string
}

type FetchRequest struct {
	ServerAddress string
	Hashes        []hash.Hash
}

type FetchResponse common.StatusResponse

type GetConfigurationRequest struct {
}

type GetConfigurationResponse Configuration

type PollRequest struct {
	HaveGeneration uint64
}

type PollResponse struct {
	NetworkSpeed     uint64
	FetchInProgress  bool // Fetch() and Update() are mutually exclusive.
	UpdateInProgress bool
	GenerationCount  uint64
	FileSystem       *scanner.FileSystem
}

type SetConfigurationRequest Configuration

type SetConfigurationResponse common.StatusResponse

type Directory struct {
	Name string
	Mode filesystem.FileMode
	Uid  uint32
	Gid  uint32
}

type UpdateRequest struct {
	PathsToDelete     []string
	DirectoriesToMake []Directory
}

type UpdateResponse struct{}
<commit_msg>Add Triggers to proto.sub.UpdateRequest.<commit_after>package sub

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/common"
	"github.com/Symantec/Dominator/sub/scanner"
)

type Configuration struct {
	ScanSpeedPercent    uint
	NetworkSpeedPercent uint
	ScanExclusionList   []string
}

type FetchRequest struct {
	ServerAddress string
	Hashes        []hash.Hash
}

type FetchResponse common.StatusResponse

type GetConfigurationRequest struct {
}

type GetConfigurationResponse Configuration

type PollRequest struct {
	HaveGeneration uint64
}

type PollResponse struct {
	NetworkSpeed     uint64
	FetchInProgress  bool // Fetch() and Update() are mutually exclusive.
	UpdateInProgress bool
	GenerationCount  uint64
	FileSystem       *scanner.FileSystem
}

type SetConfigurationRequest Configuration

type SetConfigurationResponse common.StatusResponse

type Directory struct {
	Name string
	Mode filesystem.FileMode
	Uid  uint32
	Gid  uint32
}

type UpdateRequest struct {
	PathsToDelete     []string
	DirectoriesToMake []Directory
	Triggers          *triggers.Triggers
}

type UpdateResponse struct{}
<|endoftext|>


<commit_before>package index

import (
	"time"

	_ "github.com/go-sql-driver/mysql"
)

import (
	"encoding/json"
	"fmt"
	"log"
	"pilosa/config"
	"pilosa/util"
	"sort"
)

type Pair struct {
	Key, Count uint64
}
type Rank struct {
	*Pair
	bitmap   IBitmap
	category uint64
}

type RankList []*Rank

func (p RankList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p RankList) Len() int           { return len(p) }
func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }

type Brand struct {
	bitmap_cache     map[uint64]*Rank
	db               string
	frame            string
	slice            int
	storage          Storage
	rankings         RankList
	rank_count       int
	threshold_value  uint64
	threshold_length int
	threshold_idx    int
	skip             int
	rank_time        time.Time
}

func NewBrand(db string, frame string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
	f := new(Brand)
	f.storage = s
	f.frame = frame
	f.slice = slice
	f.db = db
	f.rank_count = 0
	f.threshold_value = 0
	f.threshold_length = threshold_len
	f.threshold_idx = threshold
	f.skip = skipp
	f.Clear() //alloc the cache
	return f

}
func (self *Brand) Clear() bool {
	self.bitmap_cache = make(map[uint64]*Rank)
	return true
}
func (self *Brand) Get(bitmap_id uint64) IBitmap {
	bm, ok := self.bitmap_cache[bitmap_id]
	if ok {
		return bm.bitmap
	}
	//I should fetch the category here..need to come up with a good source
	b, filter := self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
	self.cache_it(b, bitmap_id, filter)
	return b
}

func (self *Brand) GetFilter(bitmap_id, filter uint64) IBitmap {
	b, _ := self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
	self.cache_it(b, bitmap_id, filter)
	return b
}

func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category uint64) {
	if bm.Count() >= self.threshold_value {
		self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category}
		if len(self.bitmap_cache) > self.threshold_length {
			log.Printf("RANK: %d %d %d", len(self.bitmap_cache), self.threshold_length, self.threshold_value)
			self.Rank()
			self.trim()
		}
	}
}
func (self *Brand) trim() {
	for k, item := range self.bitmap_cache {
		if item.bitmap.Count() <= self.threshold_value {
			delete(self.bitmap_cache, k)
		}
	}
	log.Printf("TRIM: %d %d", len(self.bitmap_cache), self.threshold_length)

}

func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
	bm1, ok := self.bitmap_cache[bitmap_id]
	var bm IBitmap
	if ok {
		bm = bm1.bitmap
	} else {
		bm = self.GetFilter(bitmap_id, filter) //aways overwrites what is in cass filter type
	}
	change, chunk, address := SetBit(bm, bit_pos)
	if change {
		val := chunk.Value.Block[address.BlockIndex]
		self.storage.BeginBatch()
		self.storage.StoreBlock(int64(bitmap_id), self.db, self.frame, self.slice, filter, int64(address.ChunkKey), int32(address.BlockIndex), int64(val))
		self.storage.StoreBlock(int64(bitmap_id), self.db, self.frame, self.slice, filter, COUNTER_KEY, 0, int64(bm.Count()))
		self.storage.EndBatch()
		self.rank_count++
	}
	return change
}

func (self *Brand) Rank() {
	start := time.Now()
	var list RankList
	for k, item := range self.bitmap_cache {
		//		if item.bitmap.Count() > 50 {
		list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
		//		}
	}
	sort.Sort(list)
	self.rankings = list
	if len(list) > self.threshold_idx {
		item := list[self.threshold_idx]
		self.threshold_value = item.bitmap.Count()
	} else {
		self.threshold_value = 1
	}

	//dump(self.rankings, 10)
	self.rank_count = 0
	delta := time.Since(start)
	util.SendTimer("brand_Rank", delta.Nanoseconds())
	self.rank_time = start

}

func packagePairs(r RankList) []Pair {
	res := make([]Pair, r.Len())
	//for i := 0; i < r.Len(); i++ {
	for i, v := range r {
		res[i] = Pair{v.Key, v.Count}
	}
	return res
}

func (self *Brand) Stats() interface{} {
	total := uint64(0)
	i := uint64(0)
	bit_total := uint64(0)
	for _, v := range self.bitmap_cache {
		total += uint64(v.bitmap.Len()) * uint64(256)
		i += 1
		bit_total += v.Count
	}
	avg_bytes := uint64(0)
	avg_bits := uint64(0)
	if i > 0 {
		avg_bytes = total / i
		avg_bits = bit_total / i
	}

	stats := map[string]interface{}{
		"total size of cache in bytes":       total,
		"number of bitmaps":                  len(self.bitmap_cache),
		"avg size of bitmap in space(bytes)": avg_bytes,
		"avg size of bitmap in bits":         avg_bits,
		"rank counter":                       self.rank_count,
		"threshold_value":                    self.threshold_value,
		"threshold_length":                   self.threshold_length,
		"threshold_idx":                      self.threshold_idx,
		"skip":                               self.skip}
	return stats
}
func (self *Brand) Store(bitmap_id uint64, bm IBitmap, filter uint64) {
	self.storage.Store(int64(bitmap_id), self.db, self.frame, self.slice, filter, bm.(*Bitmap))
	self.cache_it(bm, bitmap_id, filter)
}

func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []uint64) []Pair {
	if len(self.rankings) < 50 {
		self.Rank()
	} else if self.rank_count > 0 {
		last := time.Since(self.rank_time) * time.Second
		if last > 60 {
			self.Rank()
		}
	}

	is := NewIntSet()
	for _, v := range categories {
		is.Add(v)

	}

	test := self.TopNCat(src_bitmap, n, is)
	return test
}
func dump(r RankList, n int) {
	for i, v := range r {
		log.Println(i, v)
		if i > n {
			return
		}
	}
}
func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
	breakout := 500
	var (
		o       *Rank
		results RankList
	)
	counter := 0
	x := 0

	needCat := category.Size() > 0
	for i, pair := range self.rankings {

		if needCat {
			if !category.Contains(pair.category) {
				continue
			}
		}

		if counter > n {
			break
		}
		bm := Intersection(src_bitmap, pair.bitmap)
		bc := BitCount(bm)
		if bc > 0 {
			results = append(results, &Rank{&Pair{pair.Key, bc}, bm, pair.category})
			counter = counter + 1
		}
		x = i
	}

	sort.Sort(results)
	if counter < n {
		return packagePairs(results)
	}
	end := len(results) - 1
	o = results[end]
	current_threshold := o.Count

	if current_threshold <= 10 {
		return packagePairs(results)
	}

	results = append(results, o)

	for i := x + 1; i < len(self.rankings); i++ {
		counter = counter + 1
		o = self.rankings[i]

		if needCat {
			if !category.Contains(o.category) {
				continue
			}
		}

		if i > breakout {
			break
		}

		//if o.Count < current_threshold { //done
		//need something to do with the size of initianl bitmap
		if o.Count < current_threshold { //done
			break

		}

		abitmap := Intersection(src_bitmap, o.bitmap)
		bc := BitCount(abitmap)

		if bc > current_threshold {
			if results[end-1].Count > bc {
				results[end] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
				current_threshold = bc
			} else {
				results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
				sort.Sort(results)
				o = results[end]
				current_threshold = o.Count
			}
		}
	}
	return packagePairs(results[:end])
}
func (self *Brand) getFileName() string {
	base := config.GetString("fragment_base")
	if base == "" {
		base = "."
	}

	return fmt.Sprintf("%s/%s.%s.%d.json", base, self.db, self.frame, self.slice)
}

func (self *Brand) Persist() error {
	log.Println("Brand Persist:", self.getFileName())
	self.storage.FlushBatch()
	asize := len(self.bitmap_cache)

	if asize == 0 {
		log.Println("Nothing to save ", self.getFileName())
		return nil
	}
	w, err := util.Create(self.getFileName())
	if err != nil {
		log.Println("Error opening outfile ", self.getFileName())
		log.Println(err)
		return err
	}
	defer w.Close()
	defer self.storage.Close()

	var list RankList
	for k, item := range self.bitmap_cache {
		list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
	}

	sort.Sort(list)

	results := make([]uint64, asize)
	i := 0
	for _, k := range list { //   map[uint64]*Rank
		results[i] = k.Key
		i += 1
	}

	encoder := json.NewEncoder(w)
	return encoder.Encode(results)
}

func (self *Brand) Load(requestChan chan Command, f *Fragment) {
	log.Println("Brand Load")
	r, err := util.Open(self.getFileName())
	if err != nil {
		log.Println("NO Brand Init File:", self.getFileName())
		return
	}
	dec := json.NewDecoder(r)
	var keys []uint64
	if err := dec.Decode(&keys); err != nil {
		return
		//log.Println("Bad mojo")
	}
	for _, k := range keys {
		request := NewLoadRequest(k)
		requestChan <- request
		request.Response()

	}
}
<commit_msg>modifed depth for category based topn<commit_after>package index

import (
	"time"

	_ "github.com/go-sql-driver/mysql"
)

import (
	"encoding/json"
	"fmt"
	"log"
	"pilosa/config"
	"pilosa/util"
	"sort"
)

type Pair struct {
	Key, Count uint64
}
type Rank struct {
	*Pair
	bitmap   IBitmap
	category uint64
}

type RankList []*Rank

func (p RankList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p RankList) Len() int           { return len(p) }
func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }

type Brand struct {
	bitmap_cache     map[uint64]*Rank
	db               string
	frame            string
	slice            int
	storage          Storage
	rankings         RankList
	rank_count       int
	threshold_value  uint64
	threshold_length int
	threshold_idx    int
	skip             int
	rank_time        time.Time
}

func NewBrand(db string, frame string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
	f := new(Brand)
	f.storage = s
	f.frame = frame
	f.slice = slice
	f.db = db
	f.rank_count = 0
	f.threshold_value = 0
	f.threshold_length = threshold_len
	f.threshold_idx = threshold
	f.skip = skipp
	f.Clear() //alloc the cache
	return f

}
func (self *Brand) Clear() bool {
	self.bitmap_cache = make(map[uint64]*Rank)
	return true
}
func (self *Brand) Get(bitmap_id uint64) IBitmap {
	bm, ok := self.bitmap_cache[bitmap_id]
	if ok {
		return bm.bitmap
	}
	//I should fetch the category here..need to come up with a good source
	b, filter := self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
	self.cache_it(b, bitmap_id, filter)
	return b
}

func (self *Brand) GetFilter(bitmap_id, filter uint64) IBitmap {
	b, _ := self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
	self.cache_it(b, bitmap_id, filter)
	return b
}

func (self *Brand) cache_it(bm IBitmap, bitmap_id uint64, category uint64) {
	if bm.Count() >= self.threshold_value {
		self.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category}
		if len(self.bitmap_cache) > self.threshold_length {
			log.Printf("RANK: %d %d %d", len(self.bitmap_cache), self.threshold_length, self.threshold_value)
			self.Rank()
			self.trim()
		}
	}
}
func (self *Brand) trim() {
	for k, item := range self.bitmap_cache {
		if item.bitmap.Count() <= self.threshold_value {
			delete(self.bitmap_cache, k)
		}
	}
	log.Printf("TRIM: %d %d", len(self.bitmap_cache), self.threshold_length)

}

func (self *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
	bm1, ok := self.bitmap_cache[bitmap_id]
	var bm IBitmap
	if ok {
		bm = bm1.bitmap
	} else {
		bm = self.GetFilter(bitmap_id, filter) //aways overwrites what is in cass filter type
	}
	change, chunk, address := SetBit(bm, bit_pos)
	if change {
		val := chunk.Value.Block[address.BlockIndex]
		self.storage.BeginBatch()
		self.storage.StoreBlock(int64(bitmap_id), self.db, self.frame, self.slice, filter, int64(address.ChunkKey), int32(address.BlockIndex), int64(val))
		self.storage.StoreBlock(int64(bitmap_id), self.db, self.frame, self.slice, filter, COUNTER_KEY, 0, int64(bm.Count()))
		self.storage.EndBatch()
		self.rank_count++
	}
	return change
}

func (self *Brand) Rank() {
	start := time.Now()
	var list RankList
	for k, item := range self.bitmap_cache {
		//		if item.bitmap.Count() > 50 {
		list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
		//		}
	}
	sort.Sort(list)
	self.rankings = list
	if len(list) > self.threshold_idx {
		item := list[self.threshold_idx]
		self.threshold_value = item.bitmap.Count()
	} else {
		self.threshold_value = 1
	}

	//dump(self.rankings, 10)
	self.rank_count = 0
	delta := time.Since(start)
	util.SendTimer("brand_Rank", delta.Nanoseconds())
	self.rank_time = start

}

func packagePairs(r RankList) []Pair {
	res := make([]Pair, r.Len())
	//for i := 0; i < r.Len(); i++ {
	for i, v := range r {
		res[i] = Pair{v.Key, v.Count}
	}
	return res
}

func (self *Brand) Stats() interface{} {
	total := uint64(0)
	i := uint64(0)
	bit_total := uint64(0)
	for _, v := range self.bitmap_cache {
		total += uint64(v.bitmap.Len()) * uint64(256)
		i += 1
		bit_total += v.Count
	}
	avg_bytes := uint64(0)
	avg_bits := uint64(0)
	if i > 0 {
		avg_bytes = total / i
		avg_bits = bit_total / i
	}

	stats := map[string]interface{}{
		"total size of cache in bytes":       total,
		"number of bitmaps":                  len(self.bitmap_cache),
		"avg size of bitmap in space(bytes)": avg_bytes,
		"avg size of bitmap in bits":         avg_bits,
		"rank counter":                       self.rank_count,
		"threshold_value":                    self.threshold_value,
		"threshold_length":                   self.threshold_length,
		"threshold_idx":                      self.threshold_idx,
		"skip":                               self.skip}
	return stats
}
func (self *Brand) Store(bitmap_id uint64, bm IBitmap, filter uint64) {
	self.storage.Store(int64(bitmap_id), self.db, self.frame, self.slice, filter, bm.(*Bitmap))
	self.cache_it(bm, bitmap_id, filter)
}

func (self *Brand) TopN(src_bitmap IBitmap, n int, categories []uint64) []Pair {
	if len(self.rankings) < 50 {
		self.Rank()
	} else if self.rank_count > 0 {
		last := time.Since(self.rank_time) * time.Second
		if last > 60 {
			self.Rank()
		}
	}

	is := NewIntSet()
	for _, v := range categories {
		is.Add(v)

	}

	test := self.TopNCat(src_bitmap, n, is)
	return test
}
func dump(r RankList, n int) {
	for i, v := range r {
		log.Println(i, v)
		if i > n {
			return
		}
	}
}
func (self *Brand) TopNCat(src_bitmap IBitmap, n int, category *IntSet) []Pair {
	breakout := 500
	var (
		o       *Rank
		results RankList
	)
	counter := 0
	x := 0

	needCat := category.Size() > 0
	for i, pair := range self.rankings {

		if needCat {
			if !category.Contains(pair.category) {
				continue
			}
		}

		if counter > n {
			break
		}
		bm := Intersection(src_bitmap, pair.bitmap)
		bc := BitCount(bm)
		if bc > 0 {
			results = append(results, &Rank{&Pair{pair.Key, bc}, bm, pair.category})
			counter = counter + 1
		}
		x = i
	}

	sort.Sort(results)
	if counter < n {
		return packagePairs(results)
	}
	end := len(results) - 1
	o = results[end]
	current_threshold := o.Count

	if current_threshold <= 10 {
		return packagePairs(results)
	}

	results = append(results, o)

	for i := x + 1; i < len(self.rankings); i++ {
		o = self.rankings[i]

		if needCat {
			if !category.Contains(o.category) {
				continue
			}
			counter = counter + 1
		} else {
			counter = counter + 1
		}

		if counter > breakout {
			break
		}

		//if o.Count < current_threshold { //done
		//need something to do with the size of initianl bitmap
		if o.Count < current_threshold { //done
			break

		}

		abitmap := Intersection(src_bitmap, o.bitmap)
		bc := BitCount(abitmap)

		if bc > current_threshold {
			if results[end-1].Count > bc {
				results[end] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
				current_threshold = bc
			} else {
				results[end+1] = &Rank{&Pair{o.Key, bc}, abitmap, o.category}
				sort.Sort(results)
				o = results[end]
				current_threshold = o.Count
			}
		}
	}
	return packagePairs(results[:end])
}
func (self *Brand) getFileName() string {
	base := config.GetString("fragment_base")
	if base == "" {
		base = "."
	}

	return fmt.Sprintf("%s/%s.%s.%d.json", base, self.db, self.frame, self.slice)
}

func (self *Brand) Persist() error {
	log.Println("Brand Persist:", self.getFileName())
	self.storage.FlushBatch()
	asize := len(self.bitmap_cache)

	if asize == 0 {
		log.Println("Nothing to save ", self.getFileName())
		return nil
	}
	w, err := util.Create(self.getFileName())
	if err != nil {
		log.Println("Error opening outfile ", self.getFileName())
		log.Println(err)
		return err
	}
	defer w.Close()
	defer self.storage.Close()

	var list RankList
	for k, item := range self.bitmap_cache {
		list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
	}

	sort.Sort(list)

	results := make([]uint64, asize)
	i := 0
	for _, k := range list { //   map[uint64]*Rank
		results[i] = k.Key
		i += 1
	}

	encoder := json.NewEncoder(w)
	return encoder.Encode(results)
}

func (self *Brand) Load(requestChan chan Command, f *Fragment) {
	log.Println("Brand Load")
	r, err := util.Open(self.getFileName())
	if err != nil {
		log.Println("NO Brand Init File:", self.getFileName())
		return
	}
	dec := json.NewDecoder(r)
	var keys []uint64
	if err := dec.Decode(&keys); err != nil {
		return
		//log.Println("Bad mojo")
	}
	for _, k := range keys {
		request := NewLoadRequest(k)
		requestChan <- request
		request.Response()

	}
}
<|endoftext|>


<commit_before>package k8s_test

import (
	"encoding/json"
	"time"

	"github.com/onsi/gomega/gexec"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Kubernetes credential management", func() {
	var (
		proxySession *gexec.Session
		atcEndpoint  string
		username     = "test"
		password     = "test"
		extraArgs    []string
	)

	BeforeEach(func() {
		setReleaseNameAndNamespace("k8s-cm")
	})

	JustBeforeEach(func() {

		deployConcourseChart(releaseName, append([]string{"--set=worker.replicas=1"}, extraArgs...)...)

		waitAllPodsInNamespaceToBeReady(namespace)

		By("Creating the web proxy")
		proxySession, atcEndpoint = startPortForwarding(namespace, "service/"+releaseName+"-web", "8080")

		By("Logging in")
		fly.Login(username, password, atcEndpoint)

		By("Waiting for a running worker")
		Eventually(func() []Worker {
			return getRunningWorkers(fly.GetWorkers())
		}, 2*time.Minute, 10*time.Second).
			ShouldNot(HaveLen(0))
	})

	Context("/api/v1/info/creds", func() {
		var parsedResponse struct {
			Kubernetes struct {
				ConfigPath      string `json:"config_path"`
				InClusterConfig bool   `json:"in_cluster_config"`
				NamespaceConfig string `json:"namespace_config"`
			} `json:"kubernetes"`
		}

		JustBeforeEach(func() {
			token, err := FetchToken(atcEndpoint, username, password)
			Expect(err).ToNot(HaveOccurred())

			body, err := RequestCredsInfo(atcEndpoint, token.AccessToken)
			Expect(err).ToNot(HaveOccurred())

			err = json.Unmarshal(body, &parsedResponse)
			Expect(err).ToNot(HaveOccurred())
		})

		It("Contains kubernetes config", func() {
			Expect(parsedResponse.Kubernetes.ConfigPath).To(BeEmpty())
			Expect(parsedResponse.Kubernetes.InClusterConfig).To(BeTrue())
			Expect(parsedResponse.Kubernetes.NamespaceConfig).To(Equal(releaseName + "-"))
		})
	})

	Context("Consuming per-team k8s secrets", func() {
		JustBeforeEach(func() {
			// ((foo)) --> bar
			createCredentialSecret(releaseName, "foo", "main", map[string]string{"value": "bar"})

			// ((caz.baz)) --> zaz
			createCredentialSecret(releaseName, "caz", "main", map[string]string{"baz": "zaz"})

			fly.Run("set-pipeline", "-n", "-c", "../pipelines/minimal-credential-management.yml", "-p", "pipeline")
			session := fly.Start("get-pipeline", "-p", "pipeline")
			Wait(session)

			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).ToNot(ContainSubstring("zaz"))

			fly.Run("unpause-pipeline", "-p", "pipeline")
		})

		Context("using the default namespace created by the chart", func() {
			It("Gets credentials set by consuming k8s secrets", func() {
				session := fly.Start("trigger-job", "-j", "pipeline/unit", "-w")
				Wait(session)

				Expect(string(session.Out.Contents())).To(ContainSubstring("bar"))
				Expect(string(session.Out.Contents())).To(ContainSubstring("zaz"))
			})
		})

		Context("using a user-provided namespace", func() {
			BeforeEach(func() {
				Run(nil, "kubectl", "create", "namespace", releaseName+"-main")
				extraArgs = []string{
					"--set=concourse.web.kubernetes.createTeamNamespaces=false",
				}
			})

			It("Gets credentials set by consuming k8s secrets", func() {
				session := fly.Start("trigger-job", "-j", "pipeline/unit", "-w")
				Wait(session)

				Expect(string(session.Out.Contents())).To(ContainSubstring("bar"))
				Expect(string(session.Out.Contents())).To(ContainSubstring("zaz"))
			})

			AfterEach(func() {
				Run(nil, "kubectl", "delete", "namespace", releaseName+"-main", "--wait=false")
			})
		})
	})

	AfterEach(func() {
		cleanup(releaseName, namespace, proxySession)
	})

})

func createCredentialSecret(releaseName, secretName, team string, kv map[string]string) {
	args := []string{
		"create",
		"secret",
		"generic",
		secretName,
		"--namespace=" + releaseName + "-" + team,
	}

	for key, value := range kv {
		args = append(args, "--from-literal="+key+"="+value)
	}

	Run(nil, "kubectl", args...)
}
<commit_msg>k8s/topgun: test k8s credential management caching<commit_after>package k8s_test

import (
	"encoding/json"
	"time"

	"github.com/onsi/gomega/gexec"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Kubernetes credential management", func() {
	var (
		proxySession *gexec.Session
		atcEndpoint  string
		username     = "test"
		password     = "test"
		extraArgs    []string
	)

	BeforeEach(func() {
		setReleaseNameAndNamespace("k8s-cm")
	})

	JustBeforeEach(func() {

		deployConcourseChart(releaseName, append([]string{
			"--set=worker.replicas=1",
		}, extraArgs...)...)

		waitAllPodsInNamespaceToBeReady(namespace)

		By("Creating the web proxy")
		proxySession, atcEndpoint = startPortForwarding(namespace, "service/"+releaseName+"-web", "8080")

		By("Logging in")
		fly.Login(username, password, atcEndpoint)

		By("Waiting for a running worker")
		Eventually(func() []Worker {
			return getRunningWorkers(fly.GetWorkers())
		}, 2*time.Minute, 10*time.Second).
			ShouldNot(HaveLen(0))
	})

	Context("/api/v1/info/creds", func() {
		var parsedResponse struct {
			Kubernetes struct {
				ConfigPath      string `json:"config_path"`
				InClusterConfig bool   `json:"in_cluster_config"`
				NamespaceConfig string `json:"namespace_config"`
			} `json:"kubernetes"`
		}

		JustBeforeEach(func() {
			token, err := FetchToken(atcEndpoint, username, password)
			Expect(err).ToNot(HaveOccurred())

			body, err := RequestCredsInfo(atcEndpoint, token.AccessToken)
			Expect(err).ToNot(HaveOccurred())

			err = json.Unmarshal(body, &parsedResponse)
			Expect(err).ToNot(HaveOccurred())
		})

		It("Contains kubernetes config", func() {
			Expect(parsedResponse.Kubernetes.ConfigPath).To(BeEmpty())
			Expect(parsedResponse.Kubernetes.InClusterConfig).To(BeTrue())
			Expect(parsedResponse.Kubernetes.NamespaceConfig).To(Equal(releaseName + "-"))
		})
	})

	Context("Consuming per-team k8s secrets", func() {

		JustBeforeEach(func() {
			// ((foo)) --> bar
			createCredentialSecret(releaseName, "foo", "main", map[string]string{"value": "bar"})

			// ((caz.baz)) --> zaz
			createCredentialSecret(releaseName, "caz", "main", map[string]string{"baz": "zaz"})

			fly.Run("set-pipeline", "-n",
				"-c", "../pipelines/minimal-credential-management.yml",
				"-p", "pipeline",
			)

			fly.Run("unpause-pipeline", "-p", "pipeline")

		})

		var runsBuildWithCredentialsResolved = func() {
			session := fly.Start("trigger-job", "-j", "pipeline/unit", "-w")
			Wait(session)

			Expect(string(session.Out.Contents())).To(ContainSubstring("bar"))
			Expect(string(session.Out.Contents())).To(ContainSubstring("zaz"))
		}

		Context("using the default namespace created by the chart", func() {
			It("succeeds", runsBuildWithCredentialsResolved)
		})

		Context("with caching enabled", func() {
			BeforeEach(func() {
				extraArgs = []string{
					"--set=concourse.web.secretCacheEnabled=true",
					"--set=concourse.web.secretCacheDuration=600",
				}
			})

			It("gets cached credentials", func() {
				runsBuildWithCredentialsResolved()
				deleteSecret(releaseName, "main", "foo")
				deleteSecret(releaseName, "main", "caz")
				runsBuildWithCredentialsResolved()
			})
		})

		Context("using a user-provided namespace", func() {
			BeforeEach(func() {
				Run(nil, "kubectl", "create", "namespace", releaseName+"-main")
				extraArgs = []string{
					"--set=concourse.web.kubernetes.createTeamNamespaces=false",
				}
			})

			It("succeeds", runsBuildWithCredentialsResolved)

			AfterEach(func() {
				Run(nil, "kubectl", "delete", "namespace", releaseName+"-main", "--wait=false")
			})
		})
	})

	AfterEach(func() {
		cleanup(releaseName, namespace, proxySession)
	})

})

func deleteSecret(releaseName, team, secretName string) {
	Run(nil, "kubectl", "--namespace="+releaseName+"-main", "delete", "secret", secretName)
}

func createCredentialSecret(releaseName, secretName, team string, kv map[string]string) {
	args := []string{
		"create",
		"secret",
		"generic",
		secretName,
		"--namespace=" + releaseName + "-" + team,
	}

	for key, value := range kv {
		args = append(args, "--from-literal="+key+"="+value)
	}

	Run(nil, "kubectl", args...)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unversioned

import (
	"errors"
	"net/http"
)

// KubeletClient is an interface for all kubelet functionality
type KubeletClient interface {
	ConnectionInfoGetter
}

type ConnectionInfoGetter interface {
	GetConnectionInfo(host string) (scheme string, port uint, transport http.RoundTripper, err error)
}

// HTTPKubeletClient is the default implementation of KubeletHealthchecker, accesses the kubelet over HTTP.
type HTTPKubeletClient struct {
	Client *http.Client
	Config *KubeletConfig
}

func MakeTransport(config *KubeletConfig) (http.RoundTripper, error) {
	cfg := &Config{TLSClientConfig: config.TLSClientConfig}
	if config.EnableHttps {
		hasCA := len(config.CAFile) > 0 || len(config.CAData) > 0
		if !hasCA {
			cfg.Insecure = true
		}
	}
	tlsConfig, err := TLSConfigFor(cfg)
	if err != nil {
		return nil, err
	}

	transport := http.DefaultTransport
	if config.Dial != nil || tlsConfig != nil {
		transport = &http.Transport{
			Dial:            config.Dial,
			TLSClientConfig: tlsConfig,
		}
	}

	if len(config.BearerToken) > 0 {
		transport = NewBearerAuthRoundTripper(config.BearerToken, transport)
	}

	return transport, nil
}

// TODO: this structure is questionable, it should be using client.Config and overriding defaults.
func NewKubeletClient(config *KubeletConfig) (KubeletClient, error) {
	transport, err := MakeTransport(config)
	if err != nil {
		return nil, err
	}
	c := &http.Client{
		Transport: transport,
		Timeout:   config.HTTPTimeout,
	}
	return &HTTPKubeletClient{
		Client: c,
		Config: config,
	}, nil
}

func (c *HTTPKubeletClient) GetConnectionInfo(host string) (string, uint, http.RoundTripper, error) {
	scheme := "http"
	if c.Config.EnableHttps {
		scheme = "https"
	}
	return scheme, c.Config.Port, c.Client.Transport, nil
}

// FakeKubeletClient is a fake implementation of KubeletClient which returns an error
// when called.  It is useful to pass to the master in a test configuration with
// no kubelets.
type FakeKubeletClient struct{}

func (c FakeKubeletClient) GetConnectionInfo(host string) (string, uint, http.RoundTripper, error) {
	return "", 0, nil, errors.New("Not Implemented")
}
<commit_msg>UPSTREAM: 14967: Add util to set transport defaults<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unversioned

import (
	"errors"
	"net/http"

	"k8s.io/kubernetes/pkg/util"
)

// KubeletClient is an interface for all kubelet functionality
type KubeletClient interface {
	ConnectionInfoGetter
}

type ConnectionInfoGetter interface {
	GetConnectionInfo(host string) (scheme string, port uint, transport http.RoundTripper, err error)
}

// HTTPKubeletClient is the default implementation of KubeletHealthchecker, accesses the kubelet over HTTP.
type HTTPKubeletClient struct {
	Client *http.Client
	Config *KubeletConfig
}

func MakeTransport(config *KubeletConfig) (http.RoundTripper, error) {
	cfg := &Config{TLSClientConfig: config.TLSClientConfig}
	if config.EnableHttps {
		hasCA := len(config.CAFile) > 0 || len(config.CAData) > 0
		if !hasCA {
			cfg.Insecure = true
		}
	}
	tlsConfig, err := TLSConfigFor(cfg)
	if err != nil {
		return nil, err
	}

	transport := http.DefaultTransport
	if config.Dial != nil || tlsConfig != nil {
		transport = util.SetTransportDefaults(&http.Transport{
			Dial:            config.Dial,
			TLSClientConfig: tlsConfig,
		})
	}

	if len(config.BearerToken) > 0 {
		transport = NewBearerAuthRoundTripper(config.BearerToken, transport)
	}

	return transport, nil
}

// TODO: this structure is questionable, it should be using client.Config and overriding defaults.
func NewKubeletClient(config *KubeletConfig) (KubeletClient, error) {
	transport, err := MakeTransport(config)
	if err != nil {
		return nil, err
	}
	c := &http.Client{
		Transport: transport,
		Timeout:   config.HTTPTimeout,
	}
	return &HTTPKubeletClient{
		Client: c,
		Config: config,
	}, nil
}

func (c *HTTPKubeletClient) GetConnectionInfo(host string) (string, uint, http.RoundTripper, error) {
	scheme := "http"
	if c.Config.EnableHttps {
		scheme = "https"
	}
	return scheme, c.Config.Port, c.Client.Transport, nil
}

// FakeKubeletClient is a fake implementation of KubeletClient which returns an error
// when called.  It is useful to pass to the master in a test configuration with
// no kubelets.
type FakeKubeletClient struct{}

func (c FakeKubeletClient) GetConnectionInfo(host string) (string, uint, http.RoundTripper, error) {
	return "", 0, nil, errors.New("Not Implemented")
}
<|endoftext|>


<commit_before>package native

import (
	"os"
	"runtime"

	"github.com/go-delve/delve/pkg/proc"
)

// Process represents all of the information the debugger
// is holding onto regarding the process we are debugging.
type nativeProcess struct {
	bi *proc.BinaryInfo

	pid int // Process Pid

	// Breakpoint table, holds information on breakpoints.
	// Maps instruction address to Breakpoint struct.
	breakpoints proc.BreakpointMap

	// List of threads mapped as such: pid -> *Thread
	threads map[int]*nativeThread

	// Thread used to read and write memory
	memthread *nativeThread

	os             *osProcessDetails
	firstStart     bool
	ptraceChan     chan func()
	ptraceDoneChan chan interface{}
	childProcess   bool // this process was launched, not attached to

	// Controlling terminal file descriptor for
	// this process.
	ctty *os.File

	iscgo bool

	exited, detached bool
}

// newProcess returns an initialized Process struct. Before returning,
// it will also launch a goroutine in order to handle ptrace(2)
// functions. For more information, see the documentation on
// `handlePtraceFuncs`.
func newProcess(pid int) *nativeProcess {
	dbp := &nativeProcess{
		pid:            pid,
		threads:        make(map[int]*nativeThread),
		breakpoints:    proc.NewBreakpointMap(),
		firstStart:     true,
		os:             new(osProcessDetails),
		ptraceChan:     make(chan func()),
		ptraceDoneChan: make(chan interface{}),
		bi:             proc.NewBinaryInfo(runtime.GOOS, runtime.GOARCH),
	}
	go dbp.handlePtraceFuncs()
	return dbp
}

// BinInfo will return the binary info struct associated with this process.
func (dbp *nativeProcess) BinInfo() *proc.BinaryInfo {
	return dbp.bi
}

// StartCallInjection notifies the backend that we are about to inject a function call.
func (dbp *nativeProcess) StartCallInjection() (func(), error) { return func() {}, nil }

// Detach from the process being debugged, optionally killing it.
func (dbp *nativeProcess) Detach(kill bool) (err error) {
	if dbp.exited {
		return nil
	}
	if kill && dbp.childProcess {
		err := dbp.kill()
		if err != nil {
			return err
		}
		dbp.bi.Close()
		return nil
	}
	dbp.execPtraceFunc(func() {
		err = dbp.detach(kill)
		if err != nil {
			return
		}
		if kill {
			err = killProcess(dbp.pid)
		}
	})
	dbp.detached = true
	dbp.postExit()
	return
}

// Valid returns whether the process is still attached to and
// has not exited.
func (dbp *nativeProcess) Valid() (bool, error) {
	if dbp.detached {
		return false, proc.ErrProcessDetached
	}
	if dbp.exited {
		return false, proc.ErrProcessExited{Pid: dbp.pid}
	}
	return true, nil
}

// ThreadList returns a list of threads in the process.
func (dbp *nativeProcess) ThreadList() []proc.Thread {
	r := make([]proc.Thread, 0, len(dbp.threads))
	for _, v := range dbp.threads {
		r = append(r, v)
	}
	return r
}

// FindThread attempts to find the thread with the specified ID.
func (dbp *nativeProcess) FindThread(threadID int) (proc.Thread, bool) {
	th, ok := dbp.threads[threadID]
	return th, ok
}

// Memory returns the process memory.
func (dbp *nativeProcess) Memory() proc.MemoryReadWriter {
	return dbp.memthread
}

// Breakpoints returns a list of breakpoints currently set.
func (dbp *nativeProcess) Breakpoints() *proc.BreakpointMap {
	return &dbp.breakpoints
}

// RequestManualStop sets the `manualStopRequested` flag and
// sends SIGSTOP to all threads.
func (dbp *nativeProcess) RequestManualStop(cctx *proc.ContinueOnceContext) error {
	if dbp.exited {
		return proc.ErrProcessExited{Pid: dbp.pid}
	}
	return dbp.requestManualStop()
}

func (dbp *nativeProcess) WriteBreakpoint(bp *proc.Breakpoint) error {
	if bp.WatchType != 0 {
		for _, thread := range dbp.threads {
			err := thread.writeHardwareBreakpoint(bp.Addr, bp.WatchType, bp.HWBreakIndex)
			if err != nil {
				return err
			}
		}
		return nil
	}

	bp.OriginalData = make([]byte, dbp.bi.Arch.BreakpointSize())
	_, err := dbp.memthread.ReadMemory(bp.OriginalData, bp.Addr)
	if err != nil {
		return err
	}
	return dbp.writeSoftwareBreakpoint(dbp.memthread, bp.Addr)
}

func (dbp *nativeProcess) EraseBreakpoint(bp *proc.Breakpoint) error {
	if bp.WatchType != 0 {
		for _, thread := range dbp.threads {
			err := thread.clearHardwareBreakpoint(bp.Addr, bp.WatchType, bp.HWBreakIndex)
			if err != nil {
				return err
			}
		}
		return nil
	}

	return dbp.memthread.clearSoftwareBreakpoint(bp)
}

// ContinueOnce will continue the target until it stops.
// This could be the result of a breakpoint or signal.
func (dbp *nativeProcess) ContinueOnce(cctx *proc.ContinueOnceContext) (proc.Thread, proc.StopReason, error) {
	if dbp.exited {
		return nil, proc.StopExited, proc.ErrProcessExited{Pid: dbp.pid}
	}

	for {

		if err := dbp.resume(); err != nil {
			return nil, proc.StopUnknown, err
		}

		for _, th := range dbp.threads {
			th.CurrentBreakpoint.Clear()
		}

		if cctx.ResumeChan != nil {
			close(cctx.ResumeChan)
			cctx.ResumeChan = nil
		}

		trapthread, err := dbp.trapWait(-1)
		if err != nil {
			return nil, proc.StopUnknown, err
		}
		trapthread, err = dbp.stop(cctx, trapthread)
		if err != nil {
			return nil, proc.StopUnknown, err
		}
		if trapthread != nil {
			dbp.memthread = trapthread
			return trapthread, proc.StopUnknown, nil
		}
	}
}

// FindBreakpoint finds the breakpoint for the given pc.
func (dbp *nativeProcess) FindBreakpoint(pc uint64, adjustPC bool) (*proc.Breakpoint, bool) {
	if adjustPC {
		// Check to see if address is past the breakpoint, (i.e. breakpoint was hit).
		if bp, ok := dbp.breakpoints.M[pc-uint64(dbp.bi.Arch.BreakpointSize())]; ok {
			return bp, true
		}
	}
	// Directly use addr to lookup breakpoint.
	if bp, ok := dbp.breakpoints.M[pc]; ok {
		return bp, true
	}
	return nil, false
}

// initialize will ensure that all relevant information is loaded
// so the process is ready to be debugged.
func (dbp *nativeProcess) initialize(path string, debugInfoDirs []string) (*proc.Target, error) {
	if err := initialize(dbp); err != nil {
		return nil, err
	}
	if err := dbp.updateThreadList(); err != nil {
		return nil, err
	}
	stopReason := proc.StopLaunched
	if !dbp.childProcess {
		stopReason = proc.StopAttached
	}
	tgt, err := proc.NewTarget(dbp, dbp.pid, dbp.memthread, proc.NewTargetConfig{
		Path:          path,
		DebugInfoDirs: debugInfoDirs,

		// We disable asyncpreempt for the following reasons:
		//  - on Windows asyncpreempt is incompatible with debuggers, see:
		//    https://github.com/golang/go/issues/36494
		//  - freebsd's backend is generally broken and asyncpreempt makes it even more so, see:
		//    https://github.com/go-delve/delve/issues/1754
		//  - on linux/arm64 asyncpreempt can sometimes restart a sequence of
		//    instructions, if the sequence happens to contain a breakpoint it will
		//    look like the breakpoint was hit twice when it was "logically" only
		//    executed once.
		//    See: https://go-review.googlesource.com/c/go/+/208126
		DisableAsyncPreempt: runtime.GOOS == "windows" || runtime.GOOS == "freebsd" || (runtime.GOOS == "linux" && runtime.GOARCH == "arm64"),

		StopReason: stopReason,
		CanDump:    runtime.GOOS == "linux" || runtime.GOOS == "windows",
	})
	if err != nil {
		return nil, err
	}
	if dbp.bi.Arch.Name == "arm64" {
		dbp.iscgo = tgt.IsCgo()
	}
	return tgt, nil
}

func (dbp *nativeProcess) handlePtraceFuncs() {
	// We must ensure here that we are running on the same thread during
	// while invoking the ptrace(2) syscall. This is due to the fact that ptrace(2) expects
	// all commands after PTRACE_ATTACH to come from the same thread.
	runtime.LockOSThread()

	for fn := range dbp.ptraceChan {
		fn()
		dbp.ptraceDoneChan <- nil
	}
}

func (dbp *nativeProcess) execPtraceFunc(fn func()) {
	dbp.ptraceChan <- fn
	<-dbp.ptraceDoneChan
}

func (dbp *nativeProcess) postExit() {
	dbp.exited = true
	close(dbp.ptraceChan)
	close(dbp.ptraceDoneChan)
	dbp.bi.Close()
	if dbp.ctty != nil {
		dbp.ctty.Close()
	}
	dbp.os.Close()
}

func (dbp *nativeProcess) writeSoftwareBreakpoint(thread *nativeThread, addr uint64) error {
	_, err := thread.WriteMemory(addr, dbp.bi.Arch.BreakpointInstruction())
	return err
}

func openRedirects(redirects [3]string, foreground bool) (stdin, stdout, stderr *os.File, closefn func(), err error) {
	toclose := []*os.File{}

	if redirects[0] != "" {
		stdin, err = os.Open(redirects[0])
		if err != nil {
			return nil, nil, nil, nil, err
		}
		toclose = append(toclose, stdin)
	} else if foreground {
		stdin = os.Stdin
	}

	create := func(path string, dflt *os.File) *os.File {
		if path == "" {
			return dflt
		}
		var f *os.File
		f, err = os.Create(path)
		if f != nil {
			toclose = append(toclose, f)
		}
		return f
	}

	stdout = create(redirects[1], os.Stdout)
	if err != nil {
		return nil, nil, nil, nil, err
	}

	stderr = create(redirects[2], os.Stderr)
	if err != nil {
		return nil, nil, nil, nil, err
	}

	closefn = func() {
		for _, f := range toclose {
			_ = f.Close()
		}
	}

	return stdin, stdout, stderr, closefn, nil
}
<commit_msg>proc: unlock OS thread on exit from handlePtraceFuncs (#2972)<commit_after>package native

import (
	"os"
	"runtime"

	"github.com/go-delve/delve/pkg/proc"
)

// Process represents all of the information the debugger
// is holding onto regarding the process we are debugging.
type nativeProcess struct {
	bi *proc.BinaryInfo

	pid int // Process Pid

	// Breakpoint table, holds information on breakpoints.
	// Maps instruction address to Breakpoint struct.
	breakpoints proc.BreakpointMap

	// List of threads mapped as such: pid -> *Thread
	threads map[int]*nativeThread

	// Thread used to read and write memory
	memthread *nativeThread

	os             *osProcessDetails
	firstStart     bool
	ptraceChan     chan func()
	ptraceDoneChan chan interface{}
	childProcess   bool // this process was launched, not attached to

	// Controlling terminal file descriptor for
	// this process.
	ctty *os.File

	iscgo bool

	exited, detached bool
}

// newProcess returns an initialized Process struct. Before returning,
// it will also launch a goroutine in order to handle ptrace(2)
// functions. For more information, see the documentation on
// `handlePtraceFuncs`.
func newProcess(pid int) *nativeProcess {
	dbp := &nativeProcess{
		pid:            pid,
		threads:        make(map[int]*nativeThread),
		breakpoints:    proc.NewBreakpointMap(),
		firstStart:     true,
		os:             new(osProcessDetails),
		ptraceChan:     make(chan func()),
		ptraceDoneChan: make(chan interface{}),
		bi:             proc.NewBinaryInfo(runtime.GOOS, runtime.GOARCH),
	}
	go dbp.handlePtraceFuncs()
	return dbp
}

// BinInfo will return the binary info struct associated with this process.
func (dbp *nativeProcess) BinInfo() *proc.BinaryInfo {
	return dbp.bi
}

// StartCallInjection notifies the backend that we are about to inject a function call.
func (dbp *nativeProcess) StartCallInjection() (func(), error) { return func() {}, nil }

// Detach from the process being debugged, optionally killing it.
func (dbp *nativeProcess) Detach(kill bool) (err error) {
	if dbp.exited {
		return nil
	}
	if kill && dbp.childProcess {
		err := dbp.kill()
		if err != nil {
			return err
		}
		dbp.bi.Close()
		return nil
	}
	dbp.execPtraceFunc(func() {
		err = dbp.detach(kill)
		if err != nil {
			return
		}
		if kill {
			err = killProcess(dbp.pid)
		}
	})
	dbp.detached = true
	dbp.postExit()
	return
}

// Valid returns whether the process is still attached to and
// has not exited.
func (dbp *nativeProcess) Valid() (bool, error) {
	if dbp.detached {
		return false, proc.ErrProcessDetached
	}
	if dbp.exited {
		return false, proc.ErrProcessExited{Pid: dbp.pid}
	}
	return true, nil
}

// ThreadList returns a list of threads in the process.
func (dbp *nativeProcess) ThreadList() []proc.Thread {
	r := make([]proc.Thread, 0, len(dbp.threads))
	for _, v := range dbp.threads {
		r = append(r, v)
	}
	return r
}

// FindThread attempts to find the thread with the specified ID.
func (dbp *nativeProcess) FindThread(threadID int) (proc.Thread, bool) {
	th, ok := dbp.threads[threadID]
	return th, ok
}

// Memory returns the process memory.
func (dbp *nativeProcess) Memory() proc.MemoryReadWriter {
	return dbp.memthread
}

// Breakpoints returns a list of breakpoints currently set.
func (dbp *nativeProcess) Breakpoints() *proc.BreakpointMap {
	return &dbp.breakpoints
}

// RequestManualStop sets the `manualStopRequested` flag and
// sends SIGSTOP to all threads.
func (dbp *nativeProcess) RequestManualStop(cctx *proc.ContinueOnceContext) error {
	if dbp.exited {
		return proc.ErrProcessExited{Pid: dbp.pid}
	}
	return dbp.requestManualStop()
}

func (dbp *nativeProcess) WriteBreakpoint(bp *proc.Breakpoint) error {
	if bp.WatchType != 0 {
		for _, thread := range dbp.threads {
			err := thread.writeHardwareBreakpoint(bp.Addr, bp.WatchType, bp.HWBreakIndex)
			if err != nil {
				return err
			}
		}
		return nil
	}

	bp.OriginalData = make([]byte, dbp.bi.Arch.BreakpointSize())
	_, err := dbp.memthread.ReadMemory(bp.OriginalData, bp.Addr)
	if err != nil {
		return err
	}
	return dbp.writeSoftwareBreakpoint(dbp.memthread, bp.Addr)
}

func (dbp *nativeProcess) EraseBreakpoint(bp *proc.Breakpoint) error {
	if bp.WatchType != 0 {
		for _, thread := range dbp.threads {
			err := thread.clearHardwareBreakpoint(bp.Addr, bp.WatchType, bp.HWBreakIndex)
			if err != nil {
				return err
			}
		}
		return nil
	}

	return dbp.memthread.clearSoftwareBreakpoint(bp)
}

// ContinueOnce will continue the target until it stops.
// This could be the result of a breakpoint or signal.
func (dbp *nativeProcess) ContinueOnce(cctx *proc.ContinueOnceContext) (proc.Thread, proc.StopReason, error) {
	if dbp.exited {
		return nil, proc.StopExited, proc.ErrProcessExited{Pid: dbp.pid}
	}

	for {

		if err := dbp.resume(); err != nil {
			return nil, proc.StopUnknown, err
		}

		for _, th := range dbp.threads {
			th.CurrentBreakpoint.Clear()
		}

		if cctx.ResumeChan != nil {
			close(cctx.ResumeChan)
			cctx.ResumeChan = nil
		}

		trapthread, err := dbp.trapWait(-1)
		if err != nil {
			return nil, proc.StopUnknown, err
		}
		trapthread, err = dbp.stop(cctx, trapthread)
		if err != nil {
			return nil, proc.StopUnknown, err
		}
		if trapthread != nil {
			dbp.memthread = trapthread
			return trapthread, proc.StopUnknown, nil
		}
	}
}

// FindBreakpoint finds the breakpoint for the given pc.
func (dbp *nativeProcess) FindBreakpoint(pc uint64, adjustPC bool) (*proc.Breakpoint, bool) {
	if adjustPC {
		// Check to see if address is past the breakpoint, (i.e. breakpoint was hit).
		if bp, ok := dbp.breakpoints.M[pc-uint64(dbp.bi.Arch.BreakpointSize())]; ok {
			return bp, true
		}
	}
	// Directly use addr to lookup breakpoint.
	if bp, ok := dbp.breakpoints.M[pc]; ok {
		return bp, true
	}
	return nil, false
}

// initialize will ensure that all relevant information is loaded
// so the process is ready to be debugged.
func (dbp *nativeProcess) initialize(path string, debugInfoDirs []string) (*proc.Target, error) {
	if err := initialize(dbp); err != nil {
		return nil, err
	}
	if err := dbp.updateThreadList(); err != nil {
		return nil, err
	}
	stopReason := proc.StopLaunched
	if !dbp.childProcess {
		stopReason = proc.StopAttached
	}
	tgt, err := proc.NewTarget(dbp, dbp.pid, dbp.memthread, proc.NewTargetConfig{
		Path:          path,
		DebugInfoDirs: debugInfoDirs,

		// We disable asyncpreempt for the following reasons:
		//  - on Windows asyncpreempt is incompatible with debuggers, see:
		//    https://github.com/golang/go/issues/36494
		//  - freebsd's backend is generally broken and asyncpreempt makes it even more so, see:
		//    https://github.com/go-delve/delve/issues/1754
		//  - on linux/arm64 asyncpreempt can sometimes restart a sequence of
		//    instructions, if the sequence happens to contain a breakpoint it will
		//    look like the breakpoint was hit twice when it was "logically" only
		//    executed once.
		//    See: https://go-review.googlesource.com/c/go/+/208126
		DisableAsyncPreempt: runtime.GOOS == "windows" || runtime.GOOS == "freebsd" || (runtime.GOOS == "linux" && runtime.GOARCH == "arm64"),

		StopReason: stopReason,
		CanDump:    runtime.GOOS == "linux" || runtime.GOOS == "windows",
	})
	if err != nil {
		return nil, err
	}
	if dbp.bi.Arch.Name == "arm64" {
		dbp.iscgo = tgt.IsCgo()
	}
	return tgt, nil
}

func (dbp *nativeProcess) handlePtraceFuncs() {
	// We must ensure here that we are running on the same thread during
	// while invoking the ptrace(2) syscall. This is due to the fact that ptrace(2) expects
	// all commands after PTRACE_ATTACH to come from the same thread.
	runtime.LockOSThread()

	// Leaving the OS thread locked currently leads to segfaults in the
	// Go runtime while running on FreeBSD and OpenBSD:
	//   https://github.com/golang/go/issues/52394
	if runtime.GOOS == "freebsd" || runtime.GOOS == "openbsd" {
		defer runtime.UnlockOSThread()
	}

	for fn := range dbp.ptraceChan {
		fn()
		dbp.ptraceDoneChan <- nil
	}
}

func (dbp *nativeProcess) execPtraceFunc(fn func()) {
	dbp.ptraceChan <- fn
	<-dbp.ptraceDoneChan
}

func (dbp *nativeProcess) postExit() {
	dbp.exited = true
	close(dbp.ptraceChan)
	close(dbp.ptraceDoneChan)
	dbp.bi.Close()
	if dbp.ctty != nil {
		dbp.ctty.Close()
	}
	dbp.os.Close()
}

func (dbp *nativeProcess) writeSoftwareBreakpoint(thread *nativeThread, addr uint64) error {
	_, err := thread.WriteMemory(addr, dbp.bi.Arch.BreakpointInstruction())
	return err
}

func openRedirects(redirects [3]string, foreground bool) (stdin, stdout, stderr *os.File, closefn func(), err error) {
	toclose := []*os.File{}

	if redirects[0] != "" {
		stdin, err = os.Open(redirects[0])
		if err != nil {
			return nil, nil, nil, nil, err
		}
		toclose = append(toclose, stdin)
	} else if foreground {
		stdin = os.Stdin
	}

	create := func(path string, dflt *os.File) *os.File {
		if path == "" {
			return dflt
		}
		var f *os.File
		f, err = os.Create(path)
		if f != nil {
			toclose = append(toclose, f)
		}
		return f
	}

	stdout = create(redirects[1], os.Stdout)
	if err != nil {
		return nil, nil, nil, nil, err
	}

	stderr = create(redirects[2], os.Stderr)
	if err != nil {
		return nil, nil, nil, nil, err
	}

	closefn = func() {
		for _, f := range toclose {
			_ = f.Close()
		}
	}

	return stdin, stdout, stderr, closefn, nil
}
<|endoftext|>


<commit_before>// simple publish <-> subscribe events processing for go
//
// Usage example
//	package main
//
//	import (
//		"fmt"
//		"time"
//
//		"github.com/vtg/pubsub"
//	)
//
//	var events = *pubsub.Hub
//
//	func main() {
//		events.Start(1)
//
//		events.Sub("CalcComplete", notifyComplete, notifyCompleteTime)
//		go runCalculation(2)
//		go runCalculation(4)
//		time.Sleep(10 * time.Second)
//	}
//
//	func runCalculation(n int) {
//		//processing some long calculation here
//		time.Sleep(2 * time.Second)
//		var r int
//		r = n * n
//		events.Pub("CalcComplete", r, time.Now())
//	}
//
//	func notifyComplete(e *pubsub.Event) {
//		fmt.Println("result: ", e.Objects[0])
//	}
//
//	func notifyCompleteTime(e *pubsub.Event) {
//		fmt.Println("result calculated at: ", e.Objects[1])
//	}
//
package pubsub

// Event structure to store event information
type Event struct {
	Name    string
	Objects []interface{}
}

type listener struct {
	event string
	funcs []func(*Event)
}

type HUB struct {
	pub       chan Event
	sub       chan listener
	listeners []listener
}

// Hub variable to share pubsub events
var Hub = &HUB{
	pub:       make(chan Event, 5),
	sub:       make(chan listener, 5),
	listeners: []listener{},
}

// Pub publish new event
func (h *HUB) Pub(name string, objects ...interface{}) {
	h.pub <- Event{name, objects}
}

// Sub subscribe for event
func (h *HUB) Sub(name string, f ...func(*Event)) {
	h.sub <- listener{name, f}
}

// Start pubsub process with max number of parallel processing
func (h *HUB) Start(num int) {
	if num == 0 {
		num = 1
	}
	go h.runSub()
	for i := 0; i < num; i++ {
		go h.runPub()
	}
}

func (h *HUB) runPub() {
	for {
		select {
		case c := <-h.pub:
			for _, v := range h.listeners {
				if v.event == c.Name {
					for _, f := range v.funcs {
						f(&c)
					}
				}
			}
		}
	}
}

func (h *HUB) runSub() {
	for {
		select {
		case s := <-h.sub:
			h.listeners = append(h.listeners, s)
		}
	}
}
<commit_msg>Remove unneeded selects<commit_after>// simple publish <-> subscribe events processing for go
//
// Usage example
//	package main
//
//	import (
//		"fmt"
//		"time"
//
//		"github.com/vtg/pubsub"
//	)
//
//	var events = *pubsub.Hub
//
//	func main() {
//		events.Start(1)
//
//		events.Sub("CalcComplete", notifyComplete, notifyCompleteTime)
//		go runCalculation(2)
//		go runCalculation(4)
//		time.Sleep(10 * time.Second)
//	}
//
//	func runCalculation(n int) {
//		//processing some long calculation here
//		time.Sleep(2 * time.Second)
//		var r int
//		r = n * n
//		events.Pub("CalcComplete", r, time.Now())
//	}
//
//	func notifyComplete(e *pubsub.Event) {
//		fmt.Println("result: ", e.Objects[0])
//	}
//
//	func notifyCompleteTime(e *pubsub.Event) {
//		fmt.Println("result calculated at: ", e.Objects[1])
//	}
//
package pubsub

// Event structure to store event information
type Event struct {
	Name    string
	Objects []interface{}
}

type listener struct {
	event string
	funcs []func(*Event)
}

type HUB struct {
	pub       chan Event
	sub       chan listener
	listeners []listener
}

// Hub variable to share pubsub events
var Hub = &HUB{
	pub:       make(chan Event, 5),
	sub:       make(chan listener, 5),
	listeners: []listener{},
}

// Pub publish new event
func (h *HUB) Pub(name string, objects ...interface{}) {
	h.pub <- Event{name, objects}
}

// Sub subscribe for event
func (h *HUB) Sub(name string, f ...func(*Event)) {
	h.sub <- listener{name, f}
}

// Start pubsub process with max number of parallel processing
func (h *HUB) Start(num int) {
	if num == 0 {
		num = 1
	}
	go h.runSub()
	for i := 0; i < num; i++ {
		go h.runPub()
	}
}

func (h *HUB) runPub() {
	for {
		c := <-h.pub
		for _, v := range h.listeners {
			if v.event == c.Name {
				for _, f := range v.funcs {
					f(&c)
				}
			}
		}
	}
}

func (h *HUB) runSub() {
	for {
		h.listeners = append(h.listeners, <-h.sub)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func isonlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func portbinding() string {
	if port := os.Getenv("PORT"); port != "" {
		return port
	}
	return "8080"
}

func discoveryhost() string {
	if host := os.Getenv("DISCOVERYURL"); host != "" {
		return host
	}
	return "http://localhost"
}

func apibasepath() string {
	return os.Getenv("APIBASEPATH")
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {
	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	populatedatasetinfo := func() {
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
	}

	if !isonlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())
		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up

		populatedatasetinfo()

		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.EnableContentEncoding = true
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  discoveryhost(),
		ApiPath:         "/swaggerdoc",
		SwaggerPath:     "/doc/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()} // you control what services are visible
	swagger.InstallSwaggerService(config)

	logger.Printf("analyser (%s) listening on port %s\n", AppID, portbinding())
	go func() {
		logger.Fatal(http.ListenAndServe(":"+portbinding(), nil))
	}()

	if !isonlyweb() {
		for {
			select {
			case <-urlchange:
				logger.Println("Received URL change notice, re-generating database analysis")
				populatedatasetinfo()
			case <-datachange:
				logger.Println("Received Data change notice, re-generating database analysis")
				populatedatasetinfo()
			case <-heartbeatchannel:
				logger.Println("Idle")
			}
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<commit_msg>add documentation about the currently naive approach to rebuild statistics upon data change<commit_after>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func isonlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func portbinding() string {
	if port := os.Getenv("PORT"); port != "" {
		return port
	}
	return "8080"
}

func discoveryhost() string {
	if host := os.Getenv("DISCOVERYURL"); host != "" {
		return host
	}
	return "http://localhost"
}

func apibasepath() string {
	return os.Getenv("APIBASEPATH")
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {
	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	populatedatasetinfo := func() {
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
	}

	if !isonlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())
		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up

		populatedatasetinfo()

		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.EnableContentEncoding = true
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  discoveryhost(),
		ApiPath:         "/swaggerdoc",
		SwaggerPath:     "/doc/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()} // you control what services are visible
	swagger.InstallSwaggerService(config)

	logger.Printf("analyser (%s) listening on port %s\n", AppID, portbinding())
	go func() {
		logger.Fatal(http.ListenAndServe(":"+portbinding(), nil))
	}()

	if !isonlyweb() {
		for {
			select {
			case <-urlchange:
				// TODO:naive approach here. If a URLChange or DataChange event is triggered,
				// the whole analytic database will be recreated. It would be better to trace
				// only the affected datasets and only create the relevant statistic.
				// In future, urlchange/datachange might contain a JSON-encoded []byte which contains
				// the affected IDs
				logger.Println("Received URL change notice, re-generating database analysis")
				populatedatasetinfo()
			case <-datachange:
				logger.Println("Received Data change notice, re-generating database analysis")
				populatedatasetinfo()
			case <-heartbeatchannel:
				logger.Println("Idle")
			}
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<|endoftext|>