<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"

	"github.com/go-gl/gl/v2.1/gl"
	"github.com/go-gl/glfw/v3.1/glfw"
)

func main() {
	x := flag.Int("x", 150, "The width of the matrix")
	y := flag.Int("y", 16, "The height of the matrix")
	mag := flag.Int("magification", 12, "Amount of pixels per cell")
	pixelSize := flag.Float64("pixelsize", 0.3, "The space per cell that will be lit")
	flag.Parse()

	banner := Banner{
		lenX:          *x,
		lenY:          *y,
		magnification: *mag,
		pixelSize:     *pixelSize,
	}
	go banner.RunInput(os.Stdin)
	banner.RunDisplay()
}

type Banner struct {
	lenX          int
	lenY          int
	magnification int
	pixelSize     float64
	buffer        []uint8
	bufferStream  chan []uint8
}

func (banner *Banner) RunDisplay() error {
	banner.buffer = make([]uint8, banner.NumPixels()*3)
	banner.bufferStream = make(chan []uint8, 1)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	if err := glfw.Init(); err != nil {
		return fmt.Errorf("Can't init GLFW: %v", err)
	}
	win, err := glfw.CreateWindow(banner.lenX*banner.magnification, banner.lenY*banner.magnification, "LED-Banner", nil, nil)
	if err != nil {
		return err
	}
	defer win.Destroy()
	win.MakeContextCurrent()
	glfw.SwapInterval(1)
	if err := gl.Init(); err != nil {
		return err
	}

	for !win.ShouldClose() {
		gl.Clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)
		w, h := win.GetFramebufferSize()
		cellSizeX, cellSizeY := float64(w)/float64(banner.lenX), float64(h)/float64(banner.lenY)
		gl.Viewport(0, 0, int32(w), int32(h))
		gl.MatrixMode(gl.PROJECTION)
		gl.LoadIdentity()
		gl.Ortho(-cellSizeX/2, float64(w)-cellSizeX/2, float64(h)-cellSizeY/2, -cellSizeY/2, -1, 1)
		gl.MatrixMode(gl.MODELVIEW)
		gl.LoadIdentity()

		select {
		case banner.buffer = <-banner.bufferStream:
		default:
		}

		pixelSize := banner.pixelSize * cellSizeX
		for x := 0; x < banner.lenX; x++ {
			for y := 0; y < banner.lenY; y++ {
				i := (y*banner.lenX + x) * 3
				gl.Color3ub(banner.buffer[i], banner.buffer[i+1], banner.buffer[i+2])

				rx := float64(x) * cellSizeX
				ry := float64(y) * cellSizeY
				gl.Begin(gl.QUADS)
				gl.Vertex2d(rx-pixelSize/2, ry-pixelSize/2)
				gl.Vertex2d(rx+pixelSize/2, ry-pixelSize/2)
				gl.Vertex2d(rx+pixelSize/2, ry+pixelSize/2)
				gl.Vertex2d(rx-pixelSize/2, ry+pixelSize/2)
				gl.End()
			}
		}

		win.SwapBuffers()
		glfw.PollEvents()
	}

	return nil
}

func (banner *Banner) NumPixels() int {
	return banner.lenX * banner.lenY
}

func (banner *Banner) RunInput(input io.Reader) {
	for {
		buf := make([]uint8, banner.NumPixels()*3)
		io.ReadFull(input, buf)
		banner.bufferStream <- buf
	}
}
<commit_msg>Preserve aspect the ratio independent from the window size<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"

	"github.com/go-gl/gl/v2.1/gl"
	"github.com/go-gl/glfw/v3.1/glfw"
)

func main() {
	x := flag.Int("x", 150, "The width of the matrix")
	y := flag.Int("y", 16, "The height of the matrix")
	mag := flag.Int("magification", 12, "Amount of pixels per cell")
	pixelSize := flag.Float64("pixelsize", 0.3, "The space per cell that will be lit")
	flag.Parse()

	banner := Banner{
		lenX:          *x,
		lenY:          *y,
		magnification: *mag,
		pixelSize:     *pixelSize,
	}
	go banner.RunInput(os.Stdin)
	banner.RunDisplay()
}

type Banner struct {
	lenX          int
	lenY          int
	magnification int
	pixelSize     float64
	buffer        []uint8
	bufferStream  chan []uint8
}

func (banner *Banner) RunDisplay() error {
	banner.buffer = make([]uint8, banner.NumPixels()*3)
	banner.bufferStream = make(chan []uint8, 1)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	if err := glfw.Init(); err != nil {
		return fmt.Errorf("Can't init GLFW: %v", err)
	}
	win, err := glfw.CreateWindow(banner.lenX*banner.magnification, banner.lenY*banner.magnification, "LED-Banner", nil, nil)
	if err != nil {
		return err
	}
	defer win.Destroy()
	win.MakeContextCurrent()
	glfw.SwapInterval(1)
	if err := gl.Init(); err != nil {
		return err
	}

	for !win.ShouldClose() {
		gl.Clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)
		w, h := win.GetFramebufferSize()
		cellSizeX, cellSizeY := float64(w)/float64(banner.lenX), float64(h)/float64(banner.lenY)
		cellSizeMin := cellSizeX
		if cellSizeY < cellSizeX {
			cellSizeMin = cellSizeY
		}
		gl.Viewport(0, 0, int32(w), int32(h))
		gl.MatrixMode(gl.PROJECTION)
		gl.LoadIdentity()
		gl.Ortho(-cellSizeX/2, float64(w)-cellSizeX/2, float64(h)-cellSizeY/2, -cellSizeY/2, -1, 1)
		gl.MatrixMode(gl.MODELVIEW)
		gl.LoadIdentity()

		select {
		case banner.buffer = <-banner.bufferStream:
		default:
		}

		pixelSize := banner.pixelSize * cellSizeX
		for x := 0; x < banner.lenX; x++ {
			for y := 0; y < banner.lenY; y++ {
				i := (y*banner.lenX + x) * 3
				gl.Color3ub(banner.buffer[i], banner.buffer[i+1], banner.buffer[i+2])

				rx := float64(x) * cellSizeMin
				ry := float64(y) * cellSizeMin
				gl.Begin(gl.QUADS)
				gl.Vertex2d(rx-pixelSize/2, ry-pixelSize/2)
				gl.Vertex2d(rx+pixelSize/2, ry-pixelSize/2)
				gl.Vertex2d(rx+pixelSize/2, ry+pixelSize/2)
				gl.Vertex2d(rx-pixelSize/2, ry+pixelSize/2)
				gl.End()
			}
		}

		win.SwapBuffers()
		glfw.PollEvents()
	}

	return nil
}

func (banner *Banner) NumPixels() int {
	return banner.lenX * banner.lenY
}

func (banner *Banner) RunInput(input io.Reader) {
	for {
		buf := make([]uint8, banner.NumPixels()*3)
		io.ReadFull(input, buf)
		banner.bufferStream <- buf
	}
}
<|endoftext|>


<commit_before>package goboots

import (
	"bytes"
	"net/http"
	"reflect"
	"text/template"
	//"time"
	"encoding/binary"
)

type IController interface {
	//Setup(app *App)
	Init(app *App)
	Run(w http.ResponseWriter, r *http.Request, params []string) interface{}
	PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{}
	// rendering
	Render(w http.ResponseWriter, r *http.Request, content interface{})
	_registerMethod(name string, method reflect.Value)
	_getMethod(name string) (reflect.Value, bool)
}

type PageContent struct {
	Title       string
	Content     string
	UserContent string
	HeadInclude string
	FootInclude string
}

type RedirectPageContent struct {
	Title   string
	Message string
	Path    string
}

type ErrorPageContent struct {
	Title        string
	ErrorTitle   string
	ErrorMessage string
	Content      string
}

type Controller struct {
	App           *App
	Path          string
	Name          string
	LayoutName    string
	Action        string
	LayoutAction  string
	Params        []string
	PageTitle     string
	Layout        string
	ContentType   string
	customMethods map[string]reflect.Value
}

func (c *Controller) Init(app *App) {
	c.App = app
	c.PageTitle = ""
	c.Layout = "default"
	c.ContentType = "text/html; charset=utf-8"
}

func (c *Controller) Run(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	// www.example.com/params[0]/params[1]/(...)params[n]
	content := &PageContent{
		Title:   c.App.Config.Name + " - " + c.PageTitle,
		Content: "content goes here",
	}
	return content
}

func (c *Controller) PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	return nil
}

func (c *Controller) Render(w http.ResponseWriter, r *http.Request, content interface{}) {
	c.render(w, r, content, "")
}

func (c *Controller) RenderWithLayout(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	c.render(w, r, content, customLayout)
}

func (c *Controller) render(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	if len(c.Layout) && len(customLayout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		w.Write(buff.Bytes())
		return
	}
	layoutName := c.Layout
	if len(customLayout) > 0 {
		layoutName = customLayout
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(layoutName, w, r)
	} else {
		layoutTpl = c.App.GetLayout(layoutName)
	}
	layoutTpl.Execute(w, content)
}

// RenderFromCache renders an entire page to the app's bytecache
func (c *Controller) RenderToCache(w http.ResponseWriter, r *http.Request, name string, content interface{}) {
	if len(c.Layout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		c.App.ByteCaches.SetCache(name, buff.Bytes())
		return
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(c.Layout, w, r)
	} else {
		layoutTpl = c.App.GetLayout(c.Layout)
	}
	var buffer bytes.Buffer
	err := layoutTpl.Execute(&buffer, content)
	__panic(err)
	c.App.ByteCaches.SetCache(name, buffer.Bytes())
}

// RenderFromCache renders an entire page from the app's bytecache
func (c *Controller) RenderFromCache(w http.ResponseWriter, r *http.Request, name string) interface{} {
	w.Write(c.App.ByteCaches.GetCache(name).Content)
	return nil
}

func (c *Controller) ParseContent(w http.ResponseWriter, r *http.Request, locTemplatePath string, content interface{}) string {
	var tpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		tpl = c.App.GetLocalizedViewTemplate(locTemplatePath, w, r)
	} else {
		tpl = c.App.GetViewTemplate(locTemplatePath)
	}

	//TODO: handle error if template is nil
	var buffer bytes.Buffer
	err := tpl.Execute(&buffer, content)
	__panic(err)
	return buffer.String()
}

func (c *Controller) Redirect(w http.ResponseWriter, path string, title string, message string) interface{} {
	tpl := c.App.GetLayout("redirect")
	//TODO: handle error if template is nil
	content := &RedirectPageContent{
		Title:   title,
		Message: message,
		Path:    path,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) PageError(w http.ResponseWriter, errorTitle string, errorMessage string, extraHTMLContent string) interface{} {
	tpl := c.App.GetLayout("error")
	//TODO: handle error if template is nil
	content := &ErrorPageContent{
		Title:        c.App.Config.Name + " - Error: " + errorTitle,
		ErrorTitle:   errorTitle,
		ErrorMessage: errorMessage,
		Content:      extraHTMLContent,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) _registerMethod(name string, method reflect.Value) {
	if c.customMethods == nil {
		c.customMethods = make(map[string]reflect.Value, 0)
	}
	c.customMethods[name] = method
}
func (c *Controller) _getMethod(name string) (reflect.Value, bool) {
	if c.customMethods == nil {
		return reflect.Value{}, false
	}
	val, ok := c.customMethods[name]
	return val, ok
}
<commit_msg>Setup a renderWithLayout function<commit_after>package goboots

import (
	"bytes"
	"net/http"
	"reflect"
	"text/template"
	//"time"
	"encoding/binary"
)

type IController interface {
	//Setup(app *App)
	Init(app *App)
	Run(w http.ResponseWriter, r *http.Request, params []string) interface{}
	PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{}
	// rendering
	Render(w http.ResponseWriter, r *http.Request, content interface{})
	_registerMethod(name string, method reflect.Value)
	_getMethod(name string) (reflect.Value, bool)
}

type PageContent struct {
	Title       string
	Content     string
	UserContent string
	HeadInclude string
	FootInclude string
}

type RedirectPageContent struct {
	Title   string
	Message string
	Path    string
}

type ErrorPageContent struct {
	Title        string
	ErrorTitle   string
	ErrorMessage string
	Content      string
}

type Controller struct {
	App           *App
	Path          string
	Name          string
	LayoutName    string
	Action        string
	LayoutAction  string
	Params        []string
	PageTitle     string
	Layout        string
	ContentType   string
	customMethods map[string]reflect.Value
}

func (c *Controller) Init(app *App) {
	c.App = app
	c.PageTitle = ""
	c.Layout = "default"
	c.ContentType = "text/html; charset=utf-8"
}

func (c *Controller) Run(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	// www.example.com/params[0]/params[1]/(...)params[n]
	content := &PageContent{
		Title:   c.App.Config.Name + " - " + c.PageTitle,
		Content: "content goes here",
	}
	return content
}

func (c *Controller) PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	return nil
}

func (c *Controller) Render(w http.ResponseWriter, r *http.Request, content interface{}) {
	c.render(w, r, content, "")
}

func (c *Controller) RenderWithLayout(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	c.render(w, r, content, customLayout)
}

func (c *Controller) render(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	if len(c.Layout) == 0 && len(customLayout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		w.Write(buff.Bytes())
		return
	}
	layoutName := c.Layout
	if len(customLayout) > 0 {
		layoutName = customLayout
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(layoutName, w, r)
	} else {
		layoutTpl = c.App.GetLayout(layoutName)
	}
	layoutTpl.Execute(w, content)
}

// RenderFromCache renders an entire page to the app's bytecache
func (c *Controller) RenderToCache(w http.ResponseWriter, r *http.Request, name string, content interface{}) {
	if len(c.Layout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		c.App.ByteCaches.SetCache(name, buff.Bytes())
		return
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(c.Layout, w, r)
	} else {
		layoutTpl = c.App.GetLayout(c.Layout)
	}
	var buffer bytes.Buffer
	err := layoutTpl.Execute(&buffer, content)
	__panic(err)
	c.App.ByteCaches.SetCache(name, buffer.Bytes())
}

// RenderFromCache renders an entire page from the app's bytecache
func (c *Controller) RenderFromCache(w http.ResponseWriter, r *http.Request, name string) interface{} {
	w.Write(c.App.ByteCaches.GetCache(name).Content)
	return nil
}

func (c *Controller) ParseContent(w http.ResponseWriter, r *http.Request, locTemplatePath string, content interface{}) string {
	var tpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		tpl = c.App.GetLocalizedViewTemplate(locTemplatePath, w, r)
	} else {
		tpl = c.App.GetViewTemplate(locTemplatePath)
	}

	//TODO: handle error if template is nil
	var buffer bytes.Buffer
	err := tpl.Execute(&buffer, content)
	__panic(err)
	return buffer.String()
}

func (c *Controller) Redirect(w http.ResponseWriter, path string, title string, message string) interface{} {
	tpl := c.App.GetLayout("redirect")
	//TODO: handle error if template is nil
	content := &RedirectPageContent{
		Title:   title,
		Message: message,
		Path:    path,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) PageError(w http.ResponseWriter, errorTitle string, errorMessage string, extraHTMLContent string) interface{} {
	tpl := c.App.GetLayout("error")
	//TODO: handle error if template is nil
	content := &ErrorPageContent{
		Title:        c.App.Config.Name + " - Error: " + errorTitle,
		ErrorTitle:   errorTitle,
		ErrorMessage: errorMessage,
		Content:      extraHTMLContent,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) _registerMethod(name string, method reflect.Value) {
	if c.customMethods == nil {
		c.customMethods = make(map[string]reflect.Value, 0)
	}
	c.customMethods[name] = method
}
func (c *Controller) _getMethod(name string) (reflect.Value, bool) {
	if c.customMethods == nil {
		return reflect.Value{}, false
	}
	val, ok := c.customMethods[name]
	return val, ok
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"
	"sync"
	"time"

	drive "github.com/odeke-em/google-api-go-client/drive/v2"
)

type keyValue struct {
	key   string
	value interface{}
}

func (g *Commands) Stat() error {
	channelMap := make(map[int]chan *keyValue)
	var wg sync.WaitGroup
	wg.Add(len(g.opts.Sources))
	for i, relToRootPath := range g.opts.Sources {
		go func(id int, p string, chanMap *map[int]chan *keyValue, wgg *sync.WaitGroup) {
			defer wgg.Done()
			chMap := *chanMap

			file, err := g.rem.FindByPath(p)
			if err == nil {
				chMap[id] = g.stat(p, file)
				return
			}

			fmt.Printf("%s: %v\n", p, err)
			childChan := make(chan *keyValue)
			close(childChan)
			chMap[id] = childChan
			return
		}(i, relToRootPath, &channelMap, &wg)
	}
	wg.Wait()

	throttle := time.Tick(1e9 / 10)
	// Spin until all the channels are drained
	for {
		if len(channelMap) < 1 {
			break
		}

		for key, childChan := range channelMap {
			select {
			case v := <-childChan:
				if v == nil { // Closed
					delete(channelMap, key)
				} else if v.value != nil {
					err := v.value.(error)
					if err != nil {
						fmt.Printf("v: %s err: %v\n", v.key, err)
					}
				}
			default:
			}
		}

		// Pause for a bit
		<-throttle
	}
	return nil
}

func prettyPermission(perm *drive.Permission) {
	fmt.Printf("\n*\nName: %v <%s>\n", perm.Name, perm.EmailAddress)
	kvList := []*keyValue{
		&keyValue{"Role", perm.Role},
		&keyValue{"AccountType", perm.Type},
	}
	for _, kv := range kvList {
		fmt.Printf("%-20s %-30v\n", kv.key, kv.value.(string))
	}
	fmt.Printf("*\n")
}

func prettyFileStat(relToRootPath string, file *File) {
	dirType := "file"
	if file.IsDir {
		dirType = "folder"
	}

	fmt.Printf("\n\033[92m%s\033[00m\n", relToRootPath)
	kvList := []*keyValue{
		&keyValue{"FileId", file.Id},
		&keyValue{"Bytes", fmt.Sprintf("%v", file.Size)},
		&keyValue{"Size", prettyBytes(file.Size)},
		&keyValue{"DirType", dirType},
		&keyValue{"MimeType", file.MimeType},
		&keyValue{"ModTime", fmt.Sprintf("%v", file.ModTime)},
	}
	for _, kv := range kvList {
		fmt.Printf("%-20s %-30v\n", kv.key, kv.value.(string))
	}
}

func (g *Commands) stat(relToRootPath string, file *File) chan *keyValue {
	statChan := make(chan *keyValue)

	// Arbitrary value for throttle pause duration
	throttle := time.Tick(1e9 / 5)
	go func() {
		kv := &keyValue{
			key: relToRootPath,
		}

		defer func() {
			statChan <- kv
			statChan <- nil
			close(statChan)
		}()

		prettyFileStat(relToRootPath, file)
		perms, permErr := g.rem.listPermissions(file.Id)
		if permErr != nil {
			kv.value = permErr
			return
		}
		for _, perm := range perms {
			prettyPermission(perm)
		}
		if !file.IsDir || !g.opts.Recursive {
			return
		}

		remoteChildren := g.rem.FindByParentId(file.Id, g.opts.Hidden)
		channelMap := make(map[int]chan *keyValue)
		i := 0
		for child := range remoteChildren {
			childChan := g.stat(relToRootPath+"/"+child.Name, child)
			<-throttle
			channelMap[i] = childChan
			i += 1
		}

		for {
			if len(channelMap) < 1 {
				break
			}

			for key, childChan := range channelMap {
				select {
				case v := <-childChan:
					if v == nil { // Closed
						delete(channelMap, key)
					} else {
						statChan <- v
					}
				default:
					<-throttle
				}
			}
			<-throttle
		}
	}()
	return statChan
}
<commit_msg>stat: show md5Checksum<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"
	"sync"
	"time"

	drive "github.com/odeke-em/google-api-go-client/drive/v2"
)

type keyValue struct {
	key   string
	value interface{}
}

func (g *Commands) Stat() error {
	channelMap := make(map[int]chan *keyValue)
	var wg sync.WaitGroup
	wg.Add(len(g.opts.Sources))
	for i, relToRootPath := range g.opts.Sources {
		go func(id int, p string, chanMap *map[int]chan *keyValue, wgg *sync.WaitGroup) {
			defer wgg.Done()
			chMap := *chanMap

			file, err := g.rem.FindByPath(p)
			if err == nil {
				chMap[id] = g.stat(p, file)
				return
			}

			fmt.Printf("%s: %v\n", p, err)
			childChan := make(chan *keyValue)
			close(childChan)
			chMap[id] = childChan
			return
		}(i, relToRootPath, &channelMap, &wg)
	}
	wg.Wait()

	throttle := time.Tick(1e9 / 10)
	// Spin until all the channels are drained
	for {
		if len(channelMap) < 1 {
			break
		}

		for key, childChan := range channelMap {
			select {
			case v := <-childChan:
				if v == nil { // Closed
					delete(channelMap, key)
				} else if v.value != nil {
					err := v.value.(error)
					if err != nil {
						fmt.Printf("v: %s err: %v\n", v.key, err)
					}
				}
			default:
			}
		}

		// Pause for a bit
		<-throttle
	}
	return nil
}

func prettyPermission(perm *drive.Permission) {
	fmt.Printf("\n*\nName: %v <%s>\n", perm.Name, perm.EmailAddress)
	kvList := []*keyValue{
		&keyValue{"Role", perm.Role},
		&keyValue{"AccountType", perm.Type},
	}
	for _, kv := range kvList {
		fmt.Printf("%-20s %-30v\n", kv.key, kv.value.(string))
	}
	fmt.Printf("*\n")
}

func prettyFileStat(relToRootPath string, file *File) {
	dirType := "file"
	if file.IsDir {
		dirType = "folder"
	}

	fmt.Printf("\n\033[92m%s\033[00m\n", relToRootPath)
	kvList := []*keyValue{
		&keyValue{"FileId", file.Id},
		&keyValue{"Bytes", fmt.Sprintf("%v", file.Size)},
		&keyValue{"Size", prettyBytes(file.Size)},
		&keyValue{"DirType", dirType},
		&keyValue{"MimeType", file.MimeType},
		&keyValue{"ModTime", fmt.Sprintf("%v", file.ModTime)},
	}
	if !file.IsDir {
		kvList = append(kvList, &keyValue{"Md5Checksum", file.Md5Checksum})
	}
	for _, kv := range kvList {
		fmt.Printf("%-20s %-30v\n", kv.key, kv.value.(string))
	}
}

func (g *Commands) stat(relToRootPath string, file *File) chan *keyValue {
	statChan := make(chan *keyValue)

	// Arbitrary value for throttle pause duration
	throttle := time.Tick(1e9 / 5)
	go func() {
		kv := &keyValue{
			key: relToRootPath,
		}

		defer func() {
			statChan <- kv
			statChan <- nil
			close(statChan)
		}()

		prettyFileStat(relToRootPath, file)
		perms, permErr := g.rem.listPermissions(file.Id)
		if permErr != nil {
			kv.value = permErr
			return
		}
		for _, perm := range perms {
			prettyPermission(perm)
		}
		if !file.IsDir || !g.opts.Recursive {
			return
		}

		remoteChildren := g.rem.FindByParentId(file.Id, g.opts.Hidden)
		channelMap := make(map[int]chan *keyValue)
		i := 0
		for child := range remoteChildren {
			childChan := g.stat(relToRootPath+"/"+child.Name, child)
			<-throttle
			channelMap[i] = childChan
			i += 1
		}

		for {
			if len(channelMap) < 1 {
				break
			}

			for key, childChan := range channelMap {
				select {
				case v := <-childChan:
					if v == nil { // Closed
						delete(channelMap, key)
					} else {
						statChan <- v
					}
				default:
					<-throttle
				}
			}
			<-throttle
		}
	}()
	return statChan
}
<|endoftext|>


<commit_before>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package srv

import (
	"bytes"
	"crypto/tls"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	. "github.com/uniqush/uniqush-push/push"
	"io"
	"net"
	"strconv"
	"sync"
	"time"
)

const (
	maxWaitTime time.Duration = 5
)

type pushRequest struct {
	psp *PushServiceProvider
	dp *DeliveryPoint
	notif *Notification
	mid uint32
	resChan chan<- *PushResult
}

type apnsPushService struct {
	reqChan chan *pushRequest
}

func InstallAPNS() {
	GetPushServiceManager().RegisterPushServiceType(newAPNSPushService())
}

func newAPNSPushService() *apnsPushService {
	ret := new(apnsPushService)
	ret.reqChan = make(chan *pushRequest)
	go ret.pushMux()
	return ret
}

func (p *apnsPushService) Name() string {
	return "apns"
}

func (p *apnsPushService) Finalize() {
	close(p.reqChan)
	p.reqChan = nil
}

func (p *apnsPushService) BuildPushServiceProviderFromMap(kv map[string]string, psp *PushServiceProvider) error {
	if service, ok := kv["service"]; ok {
		psp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}

	if cert, ok := kv["cert"]; ok && len(cert) > 0 {
		psp.FixedData["cert"] = cert
	} else {
		return errors.New("NoCertificate")
	}

	if key, ok := kv["key"]; ok && len(key) > 0 {
		psp.FixedData["key"] = key
	} else {
		return errors.New("NoPrivateKey")
	}

	if sandbox, ok := kv["sandbox"]; ok {
		if sandbox == "true" {
			psp.VolatileData["addr"] = "gateway.sandbox.push.apple.com:2195"
			return nil
		}
	}
	psp.VolatileData["addr"] = "gateway.push.apple.com:2195"
	return nil
}

func (p *apnsPushService) BuildDeliveryPointFromMap(kv map[string]string, dp *DeliveryPoint) error {
	if service, ok := kv["service"]; ok && len(service) > 0 {
		dp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}
	if sub, ok := kv["subscriber"]; ok && len(sub) > 0 {
		dp.FixedData["subscriber"] = sub
	} else {
		return errors.New("NoSubscriber")
	}
	if devtoken, ok := kv["devtoken"]; ok && len(devtoken) > 0 {
		if len(devtoken) != 40 {
			return fmt.Errorf("Dev token %v is invalid: it should be 40 characters", devtoken)
		}
		dp.FixedData["devtoken"] = devtoken
	} else {
		return errors.New("NoDevToken")
	}
	return nil
}

func toAPNSPayload(n *Notification) ([]byte, error) {
	payload := make(map[string]interface{})
	aps := make(map[string]interface{})
	alert := make(map[string]interface{})
	for k, v := range n.Data {
		switch k {
		case "msg":
			alert["body"] = v
		case "loc-key":
			alert[k] = v
		case "loc-args":
			alert[k] = v
		case "badge":
			b, err := strconv.Atoi(v)
			if err != nil {
				continue
			} else {
				aps["badge"] = b
			}
		case "sound":
			aps["sound"] = v
		case "img":
			alert["launch-image"] = v
		case "id":
			continue
		case "expiry":
			continue
		case "ttl":
			continue
		default:
			payload[k] = v
		}
	}
	aps["alert"] = alert
	payload["aps"] = aps
	j, err := json.Marshal(payload)
	if err != nil {
		return nil, err
	}
	return j, nil
}

func writen(w io.Writer, buf []byte) error {
	n := len(buf)
	for n >= 0 {
		l, err := w.Write(buf)
		if err != nil {
			return err
		}
		if l >= n {
			return nil
		}
		n -= l
		buf = buf[l:]
	}
	return nil
}

type apnsResult struct {
	msgId uint32
	status uint8
	err error
}

func (self *apnsPushService) Push(psp *PushServiceProvider, dpQueue <-chan *DeliveryPoint, resQueue chan<- *PushResult, notif *Notification) {
	wg := new(sync.WaitGroup)

	for dp := range dpQueue {
		wg.Add(1)
		go func () {
			resultChannel := make(chan *PushResult, 1)
			req := new(pushRequest)
			req.dp = dp
			req.notif = notif
			req.resChan = resultChannel
			req.psp = psp

			self.reqChan <- req

			for {
				select {
				case res := <-resultChannel:
					resQueue <- res
				case <-time.After(maxWaitTime * time.Second):
					return

				}
			}
			wg.Done()

		}()
	}

	wg.Wait()
	close(resQueue)
}

func (self *apnsPushService) resultCollector(psp *PushServiceProvider, resChan chan<- *apnsResult, c net.Conn) {
	for {
		var cmd uint8
		var status uint8
		var msgid uint32
		buf := make([]byte, 6)

		n, err := c.Read(buf)

		// The connection is closed by remote server. It could recover.
		if err == io.EOF {
			res := new(apnsResult)
			res.err = io.EOF
			resChan <- res
			return
		} else if err != nil {
			// Otherwise, it cannot recover.
			res := new(apnsResult)
			res.err = NewConnectionError(err)
			resChan<-res
			return
		}

		if n != 6 {
			continue
		}

		byteBuffer := bytes.NewBuffer(buf)

		err = binary.Read(byteBuffer, binary.BigEndian, &cmd)
		if err != nil {
			res := new(apnsResult)
			res.err = NewConnectionError(err)
			resChan<-res
			continue
		}

		err = binary.Read(byteBuffer, binary.BigEndian, &status)
		if err != nil {
			res := new(apnsResult)
			res.err = NewConnectionError(err)
			resChan<-res
			continue
		}

		err = binary.Read(byteBuffer, binary.BigEndian, &msgid)
		if err != nil {
			res := new(apnsResult)
			res.err = NewConnectionError(err)
			resChan<-res
			continue
		}

		res := new(apnsResult)
		res.msgId = msgid
		res.status = status
		resChan<-res
	}
}

func (self *apnsPushService) singlePush(psp *PushServiceProvider, dp *DeliveryPoint, notif *Notification, mid uint32, tlsconn net.Conn) error {
	devtoken := dp.FixedData["devtoken"]

	btoken, err := hex.DecodeString(devtoken)
	if err != nil {
		return NewBadDeliveryPointWithDetails(dp, err.Error())
	}

	bpayload, err := toAPNSPayload(notif)
	if err != nil {
		return NewBadNotificationWithDetails(err.Error())
	}

	buffer := bytes.NewBuffer([]byte{})

	// command
	binary.Write(buffer, binary.BigEndian, uint8(1))

	// transaction id
	binary.Write(buffer, binary.BigEndian, mid)

	// Expiry: default is one hour
	unixNow := uint32(time.Now().Unix())
	expiry := unixNow + 60*60
	if ttlstr, ok := notif.Data["ttl"]; ok {
		ttl, err := strconv.ParseUint(ttlstr, 10, 32)
		if err == nil {
			expiry = unixNow + uint32(ttl)
		}
	}
	binary.Write(buffer, binary.BigEndian, expiry)

	// device token
	binary.Write(buffer, binary.BigEndian, uint16(len(btoken)))
	binary.Write(buffer, binary.BigEndian, btoken)

	// payload
	binary.Write(buffer, binary.BigEndian, uint16(len(bpayload)))
	binary.Write(buffer, binary.BigEndian, bpayload)
	pdu := buffer.Bytes()

	err = writen(tlsconn, pdu)
	if err != nil {
		return NewConnectionError(err)
	}

	return nil
}

func connectAPNS(psp *PushServiceProvider) (net.Conn, error) {
	cert, err := tls.LoadX509KeyPair(psp.FixedData["cert"], psp.FixedData["key"])
	if err != nil {
		return nil, NewBadPushServiceProviderWithDetails(psp, err.Error())
	}

	conf := &tls.Config{
		Certificates:       []tls.Certificate{cert},
		//InsecureSkipVerify: true,
	}

	tlsconn, err := tls.Dial("tcp", psp.VolatileData["addr"], conf)
	if err != nil {
		return nil, NewConnectionError(err)
	}
	err = tlsconn.Handshake()
	if err != nil {
		return nil, NewConnectionError(err)
	}
	return tlsconn, nil
}

func (self *apnsPushService) pushMux() {
	connChan := make(map[string]chan *pushRequest, 10)
	for req := range self.reqChan {
		if req == nil {
			break
		}
		psp := req.psp
		var ch chan *pushRequest
		if ch, ok := connChan[psp.Name()]; !ok {
			ch = make(chan *pushRequest)
			connChan[psp.Name()] = ch
			go self.pushWorker(psp, ch)
		}
		ch <- req
	}
	for _, c := range connChan {
		close(c)
	}
}

func (self *apnsPushService) pushWorker(psp *PushServiceProvider, reqChan chan *pushRequest) {
	resChan := make(chan *apnsResult)

	reqIdMap := make(map[uint32]*pushRequest)

	var connErr error
	connErr = nil

	conn, err := connectAPNS(psp)
	if err != nil {
		connErr = err
	}

	go self.resultCollector(psp, resChan, conn)

	var nextid uint32

	nextid = 5

	for {
		select {
		case req := <-reqChan:
			// closed by another goroutine
			// close the connection and exit
			if req == nil {
				if connErr == nil {
					conn.Close()
				}
				return
			}

			dp := req.dp
			notif := req.notif
			mid := nextid
			nextid++

			if connErr != nil {
				result := new(PushResult)
				result.Content = notif
				result.Provider = psp
				result.Destination = dp
				result.MsgId = fmt.Sprintf("%v", mid)
				result.Err = connErr
				req.resChan<-result
				continue
			}

			reqIdMap[mid] = req
			err := self.singlePush(psp, dp, notif, mid, conn)

			if err != nil {
				result := new(PushResult)
				result.Content = notif
				result.Provider = psp
				result.Destination = dp
				result.MsgId = fmt.Sprintf("apns:%v-%v", psp.Name(), mid)
				result.Err = err
				req.resChan<-result
				delete(reqIdMap, mid)
			}

		case apnsres := <-resChan:
			// Connection Closed by remote server.
			// Recover it.
			if apnsres.err == io.EOF {
				conn.Close()
				conn, err = connectAPNS(psp)
				if err != nil {
					connErr = err
					continue
				}
				go self.resultCollector(psp, resChan, conn)
				continue
			}
			if cerr, ok := apnsres.err.(*ConnectionError); ok {
				connErr = cerr
			}

			if req, ok := reqIdMap[apnsres.msgId]; ok {
				result := new(PushResult)
				result.Content = req.notif
				result.Provider = psp
				result.Destination = req.dp
				result.MsgId = fmt.Sprintf("apns:%v-%v", psp.Name(), apnsres.msgId)
				if apnsres.err != nil {
					result := new(PushResult)
					result.Err = apnsres.err
					req.resChan<-result
					continue
				}

				switch apnsres.status {
				case 0:
					result.Err = nil
				case 1:
					result.Err = NewBadDeliveryPointWithDetails(req.dp, "Processing Error")
				case 2:
					result.Err = NewBadDeliveryPointWithDetails(req.dp, "Missing Device Token")
				case 3:
					result.Err = NewBadNotificationWithDetails("Missing topic")
				case 4:
					result.Err = NewBadNotificationWithDetails("Missing payload")
				case 5:
					result.Err = NewBadNotificationWithDetails("Invalid token size")
				case 6:
					result.Err = NewBadNotificationWithDetails("Invalid topic size")
				case 7:
					result.Err = NewBadNotificationWithDetails("Invalid payload size")
				case 8:
					result.Err = NewBadDeliveryPointWithDetails(req.dp, "Invalid Token")
				default:
					result.Err = fmt.Errorf("Unknown Error: %d", apnsres.status)
				}

				delete(reqIdMap, apnsres.msgId)
				go func() {
					select {
					case req.resChan<-result:
					case <-time.After((maxWaitTime + 1) * time.Second):
					}
					close(req.resChan)
				}()
			}
		}
	}
}

<commit_msg>close the channel is enough<commit_after>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package srv

import (
	"bytes"
	"crypto/tls"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	. "github.com/uniqush/uniqush-push/push"
	"io"
	"net"
	"strconv"
	"sync"
	"time"
)

const (
	maxWaitTime time.Duration = 5
)

type pushRequest struct {
	psp *PushServiceProvider
	dp *DeliveryPoint
	notif *Notification
	mid uint32
	resChan chan<- *PushResult
}

type apnsPushService struct {
	reqChan chan *pushRequest
}

func InstallAPNS() {
	GetPushServiceManager().RegisterPushServiceType(newAPNSPushService())
}

func newAPNSPushService() *apnsPushService {
	ret := new(apnsPushService)
	ret.reqChan = make(chan *pushRequest)
	go ret.pushMux()
	return ret
}

func (p *apnsPushService) Name() string {
	return "apns"
}

func (p *apnsPushService) Finalize() {
	close(p.reqChan)
}

func (p *apnsPushService) BuildPushServiceProviderFromMap(kv map[string]string, psp *PushServiceProvider) error {
	if service, ok := kv["service"]; ok {
		psp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}

	if cert, ok := kv["cert"]; ok && len(cert) > 0 {
		psp.FixedData["cert"] = cert
	} else {
		return errors.New("NoCertificate")
	}

	if key, ok := kv["key"]; ok && len(key) > 0 {
		psp.FixedData["key"] = key
	} else {
		return errors.New("NoPrivateKey")
	}

	if sandbox, ok := kv["sandbox"]; ok {
		if sandbox == "true" {
			psp.VolatileData["addr"] = "gateway.sandbox.push.apple.com:2195"
			return nil
		}
	}
	psp.VolatileData["addr"] = "gateway.push.apple.com:2195"
	return nil
}

func (p *apnsPushService) BuildDeliveryPointFromMap(kv map[string]string, dp *DeliveryPoint) error {
	if service, ok := kv["service"]; ok && len(service) > 0 {
		dp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}
	if sub, ok := kv["subscriber"]; ok && len(sub) > 0 {
		dp.FixedData["subscriber"] = sub
	} else {
		return errors.New("NoSubscriber")
	}
	if devtoken, ok := kv["devtoken"]; ok && len(devtoken) > 0 {
		if len(devtoken) != 40 {
			return fmt.Errorf("Dev token %v is invalid: it should be 40 characters", devtoken)
		}
		dp.FixedData["devtoken"] = devtoken
	} else {
		return errors.New("NoDevToken")
	}
	return nil
}

func toAPNSPayload(n *Notification) ([]byte, error) {
	payload := make(map[string]interface{})
	aps := make(map[string]interface{})
	alert := make(map[string]interface{})
	for k, v := range n.Data {
		switch k {
		case "msg":
			alert["body"] = v
		case "loc-key":
			alert[k] = v
		case "loc-args":
			alert[k] = v
		case "badge":
			b, err := strconv.Atoi(v)
			if err != nil {
				continue
			} else {
				aps["badge"] = b
			}
		case "sound":
			aps["sound"] = v
		case "img":
			alert["launch-image"] = v
		case "id":
			continue
		case "expiry":
			continue
		case "ttl":
			continue
		default:
			payload[k] = v
		}
	}
	aps["alert"] = alert
	payload["aps"] = aps
	j, err := json.Marshal(payload)
	if err != nil {
		return nil, err
	}
	return j, nil
}

func writen(w io.Writer, buf []byte) error {
	n := len(buf)
	for n >= 0 {
		l, err := w.Write(buf)
		if err != nil {
			return err
		}
		if l >= n {
			return nil
		}
		n -= l
		buf = buf[l:]
	}
	return nil
}

type apnsResult struct {
	msgId uint32
	status uint8
	err error
}

func (self *apnsPushService) Push(psp *PushServiceProvider, dpQueue <-chan *DeliveryPoint, resQueue chan<- *PushResult, notif *Notification) {
	wg := new(sync.WaitGroup)

	for dp := range dpQueue {
		wg.Add(1)
		go func () {
			resultChannel := make(chan *PushResult, 1)
			req := new(pushRequest)
			req.dp = dp
			req.notif = notif
			req.resChan = resultChannel
			req.psp = psp

			self.reqChan <- req

			for {
				select {
				case res := <-resultChannel:
					resQueue <- res
				case <-time.After(maxWaitTime * time.Second):
					return

				}
			}
			wg.Done()

		}()
	}

	wg.Wait()
	close(resQueue)
}

func (self *apnsPushService) resultCollector(psp *PushServiceProvider, resChan chan<- *apnsResult, c net.Conn) {
	for {
		var cmd uint8
		var status uint8
		var msgid uint32
		buf := make([]byte, 6)

		n, err := c.Read(buf)

		// The connection is closed by remote server. It could recover.
		if err == io.EOF {
			res := new(apnsResult)
			res.err = io.EOF
			resChan <- res
			return
		} else if err != nil {
			// Otherwise, it cannot recover.
			res := new(apnsResult)
			res.err = NewConnectionError(err)
			resChan<-res
			return
		}

		if n != 6 {
			continue
		}

		byteBuffer := bytes.NewBuffer(buf)

		err = binary.Read(byteBuffer, binary.BigEndian, &cmd)
		if err != nil {
			res.err = err
			resChan<-res
			continue
		}

		err = binary.Read(byteBuffer, binary.BigEndian, &status)
		if err != nil {
			res.err = err
			resChan<-res
			continue
		}

		err = binary.Read(byteBuffer, binary.BigEndian, &msgid)
		if err != nil {
			res.err = err
			resChan<-res
			continue
		}

		res := new(apnsResult)
		res.msgId = msgid
		res.status = status
		resChan<-res
	}
}

func (self *apnsPushService) singlePush(psp *PushServiceProvider, dp *DeliveryPoint, notif *Notification, mid uint32, tlsconn net.Conn) error {
	devtoken := dp.FixedData["devtoken"]

	btoken, err := hex.DecodeString(devtoken)
	if err != nil {
		return NewBadDeliveryPointWithDetails(dp, err.Error())
	}

	bpayload, err := toAPNSPayload(notif)
	if err != nil {
		return NewBadNotificationWithDetails(err.Error())
	}

	buffer := bytes.NewBuffer([]byte{})

	// command
	binary.Write(buffer, binary.BigEndian, uint8(1))

	// transaction id
	binary.Write(buffer, binary.BigEndian, mid)

	// Expiry: default is one hour
	unixNow := uint32(time.Now().Unix())
	expiry := unixNow + 60*60
	if ttlstr, ok := notif.Data["ttl"]; ok {
		ttl, err := strconv.ParseUint(ttlstr, 10, 32)
		if err == nil {
			expiry = unixNow + uint32(ttl)
		}
	}
	binary.Write(buffer, binary.BigEndian, expiry)

	// device token
	binary.Write(buffer, binary.BigEndian, uint16(len(btoken)))
	binary.Write(buffer, binary.BigEndian, btoken)

	// payload
	binary.Write(buffer, binary.BigEndian, uint16(len(bpayload)))
	binary.Write(buffer, binary.BigEndian, bpayload)
	pdu := buffer.Bytes()

	err = writen(tlsconn, pdu)
	if err != nil {
		return NewConnectionError(err)
	}

	return nil
}

func connectAPNS(psp *PushServiceProvider) (net.Conn, error) {
	cert, err := tls.LoadX509KeyPair(psp.FixedData["cert"], psp.FixedData["key"])
	if err != nil {
		return nil, NewBadPushServiceProviderWithDetails(psp, err.Error())
	}

	conf := &tls.Config{
		Certificates:       []tls.Certificate{cert},
		//InsecureSkipVerify: true,
	}

	tlsconn, err := tls.Dial("tcp", psp.VolatileData["addr"], conf)
	if err != nil {
		return nil, NewConnectionError(err)
	}
	err = tlsconn.Handshake()
	if err != nil {
		return nil, NewConnectionError(err)
	}
	return tlsconn, nil
}

func (self *apnsPushService) pushMux() {
	connChan := make(map[string]chan *pushRequest, 10)
	for req := range self.reqChan {
		if req == nil {
			break
		}
		psp := req.psp
		var ch chan *pushRequest
		if ch, ok := connChan[psp.Name()]; !ok {
			ch = make(chan *pushRequest)
			connChan[psp.Name()] = ch
			go self.pushWorker(psp, ch)
		}
		ch <- req
	}
	for _, c := range connChan {
		close(c)
	}
}

func (self *apnsPushService) pushWorker(psp *PushServiceProvider, reqChan chan *pushRequest) {
	resChan := make(chan *apnsResult)

	reqIdMap := make(map[uint32]*pushRequest)

	var connErr error
	connErr = nil

	conn, err := connectAPNS(psp)
	if err != nil {
		connErr = err
	}

	go self.resultCollector(psp, resChan, conn)

	var nextid uint32

	nextid = 5

	for {
		select {
		case req := <-reqChan:
			// closed by another goroutine
			// close the connection and exit
			if req == nil {
				if connErr == nil {
					conn.Close()
				}
				return
			}

			dp := req.dp
			notif := req.notif
			mid := nextid
			nextid++

			if connErr != nil {
				result := new(PushResult)
				result.Content = notif
				result.Provider = psp
				result.Destination = dp
				result.MsgId = fmt.Sprintf("%v", mid)
				result.Err = connErr
				req.resChan<-result
				continue
			}

			reqIdMap[mid] = req
			err := self.singlePush(psp, dp, notif, mid, conn)

			if err != nil {
				result := new(PushResult)
				result.Content = notif
				result.Provider = psp
				result.Destination = dp
				result.MsgId = fmt.Sprintf("apns:%v-%v", psp.Name(), mid)
				result.Err = err
				req.resChan<-result
				delete(reqIdMap, mid)
			}

		case apnsres := <-resChan:
			// Connection Closed by remote server.
			// Recover it.
			if apnsres.err == io.EOF {
				conn.Close()
				conn, err = connectAPNS(psp)
				if err != nil {
					connErr = err
					continue
				}
				go self.resultCollector(psp, resChan, conn)
				continue
			}
			if cerr, ok := apnsres.err.(*ConnectionError); ok {
				connErr = cerr
			}

			if req, ok := reqIdMap[apnsres.msgId]; ok {
				result := new(PushResult)
				result.Content = req.notif
				result.Provider = psp
				result.Destination = req.dp
				result.MsgId = fmt.Sprintf("apns:%v-%v", psp.Name(), apnsres.msgId)
				if apnsres.err != nil {
					result := new(PushResult)
					result.Err = apnsres.err
					req.resChan<-result
					continue
				}

				switch apnsres.status {
				case 0:
					result.Err = nil
				case 1:
					result.Err = NewBadDeliveryPointWithDetails(req.dp, "Processing Error")
				case 2:
					result.Err = NewBadDeliveryPointWithDetails(req.dp, "Missing Device Token")
				case 3:
					result.Err = NewBadNotificationWithDetails("Missing topic")
				case 4:
					result.Err = NewBadNotificationWithDetails("Missing payload")
				case 5:
					result.Err = NewBadNotificationWithDetails("Invalid token size")
				case 6:
					result.Err = NewBadNotificationWithDetails("Invalid topic size")
				case 7:
					result.Err = NewBadNotificationWithDetails("Invalid payload size")
				case 8:
					result.Err = NewBadDeliveryPointWithDetails(req.dp, "Invalid Token")
				default:
					result.Err = fmt.Errorf("Unknown Error: %d", apnsres.status)
				}

				delete(reqIdMap, apnsres.msgId)
				go func() {
					select {
					case req.resChan<-result:
					case <-time.After((maxWaitTime + 1) * time.Second):
					}
					close(req.resChan)
				}()
			}
		}
	}
}

<|endoftext|>


<commit_before>package sshd

import (
	"net"

	"github.com/shazow/rateio"
	"golang.org/x/crypto/ssh"
)

// SSHListener is the container for the connection and ssh-related configuration
type SSHListener struct {
	net.Listener
	config *ssh.ServerConfig

	RateLimit   func() rateio.Limiter
	HandlerFunc func(term *Terminal)
}

// ListenSSH makes an SSH listener socket
func ListenSSH(laddr string, config *ssh.ServerConfig) (*SSHListener, error) {
	socket, err := net.Listen("tcp", laddr)
	if err != nil {
		return nil, err
	}
	l := SSHListener{Listener: socket, config: config}
	return &l, nil
}

func (l *SSHListener) handleConn(conn net.Conn) (*Terminal, error) {
	if l.RateLimit != nil {
		// TODO: Configurable Limiter?
		conn = ReadLimitConn(conn, l.RateLimit())
	}

	// Upgrade TCP connection to SSH connection
	sshConn, channels, requests, err := ssh.NewServerConn(conn, l.config)
	if err != nil {
		return nil, err
	}

	// FIXME: Disconnect if too many faulty requests? (Avoid DoS.)
	go ssh.DiscardRequests(requests)
	return NewSession(sshConn, channels)
}

// Serve Accepts incoming connections as terminal requests and yield them
func (l *SSHListener) Serve() {
	defer l.Close()
	for {
		conn, err := l.Accept()

		if err != nil {
			logger.Printf("Failed to accept connection: %s", err)
			break
		}

		// Goroutineify to resume accepting sockets early
		go func() {
			term, err := l.handleConn(conn)
			if err != nil {
				logger.Printf("[%s] Failed to handshake: %s", conn.RemoteAddr(), err)
				return
			}
			l.HandlerFunc(term)
		}()
	}
}
<commit_msg>sshd: Close connection on failed handshake<commit_after>package sshd

import (
	"net"

	"github.com/shazow/rateio"
	"golang.org/x/crypto/ssh"
)

// SSHListener is the container for the connection and ssh-related configuration
type SSHListener struct {
	net.Listener
	config *ssh.ServerConfig

	RateLimit   func() rateio.Limiter
	HandlerFunc func(term *Terminal)
}

// ListenSSH makes an SSH listener socket
func ListenSSH(laddr string, config *ssh.ServerConfig) (*SSHListener, error) {
	socket, err := net.Listen("tcp", laddr)
	if err != nil {
		return nil, err
	}
	l := SSHListener{Listener: socket, config: config}
	return &l, nil
}

func (l *SSHListener) handleConn(conn net.Conn) (*Terminal, error) {
	if l.RateLimit != nil {
		// TODO: Configurable Limiter?
		conn = ReadLimitConn(conn, l.RateLimit())
	}

	// Upgrade TCP connection to SSH connection
	sshConn, channels, requests, err := ssh.NewServerConn(conn, l.config)
	if err != nil {
		return nil, err
	}

	// FIXME: Disconnect if too many faulty requests? (Avoid DoS.)
	go ssh.DiscardRequests(requests)
	return NewSession(sshConn, channels)
}

// Serve Accepts incoming connections as terminal requests and yield them
func (l *SSHListener) Serve() {
	defer l.Close()
	for {
		conn, err := l.Accept()

		if err != nil {
			logger.Printf("Failed to accept connection: %s", err)
			break
		}

		// Goroutineify to resume accepting sockets early
		go func() {
			term, err := l.handleConn(conn)
			if err != nil {
				logger.Printf("[%s] Failed to handshake: %s", conn.RemoteAddr(), err)
				conn.Close() // Must be closed to avoid a leak
				return
			}
			l.HandlerFunc(term)
		}()
	}
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"context"
	"fmt"
	"reflect"
	"runtime"
	"strconv"
	"testing"

	"github.com/apache/beam/sdks/go/pkg/beam/core/graph"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/coderx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
)

var intInput = []interface{}{int(1), int(2), int(3), int(4), int(5), int(6)}
var strInput = []interface{}{"1", "2", "3", "4", "5", "6"}

var tests = []struct {
	Fn         interface{}
	AccumCoder *coder.Coder
	Input      []interface{}
	Expected   interface{}
}{
	{Fn: mergeFn, AccumCoder: intCoder(reflectx.Int), Input: intInput, Expected: int(21)},
	{Fn: &MyCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: int(21)},
	{Fn: &MyOtherCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: "21"},
	{Fn: &MyThirdCombine{}, AccumCoder: intCoder(reflectx.Int), Input: strInput, Expected: int(21)},
	{Fn: &MyContextCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: int(21)},
	{Fn: &MyErrorCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: int(21)},
}

func fnName(x interface{}) string {
	v := reflect.ValueOf(x)
	if v.Kind() != reflect.Func {
		return v.Type().String()
	}
	return runtime.FuncForPC(uintptr(v.Pointer())).Name()
}

// TestCombine verifies that the Combine node works correctly.
func TestCombine(t *testing.T) {
	for _, test := range tests {
		t.Run(fnName(test.Fn), func(t *testing.T) {
			edge := getCombineEdge(t, test.Fn, test.AccumCoder)

			out := &CaptureNode{UID: 1}
			combine := &Combine{UID: 2, Fn: edge.CombineFn, Out: out}
			n := &FixedRoot{UID: 3, Elements: makeKeyedInput(42, test.Input...), Out: combine}

			constructAndExecutePlan(t, []Unit{n, combine, out})

			expected := makeKV(42, test.Expected)
			if !equalList(out.Elements, expected) {
				t.Errorf("combine(%s) = %v, want %v", edge.CombineFn.Name(), extractKeyedValues(out.Elements...), extractKeyedValues(expected...))
			}
		})
	}
}

// TestLiftedCombine verifies that the LiftedCombine, MergeAccumulators, and
// ExtractOutput nodes work correctly after the lift has been performed.
func TestLiftedCombine(t *testing.T) {
	for _, test := range tests {
		t.Run(fnName(test.Fn), func(t *testing.T) {
			edge := getCombineEdge(t, test.Fn, test.AccumCoder)

			out := &CaptureNode{UID: 1}
			extract := &ExtractOutput{Combine: &Combine{UID: 2, Fn: edge.CombineFn, Out: out}}
			merge := &MergeAccumulators{Combine: &Combine{UID: 3, Fn: edge.CombineFn, Out: extract}}
			gbk := &simpleGBK{UID: 4, Out: merge}
			precombine := &LiftedCombine{Combine: &Combine{UID: 5, Fn: edge.CombineFn, Out: gbk}}
			n := &FixedRoot{UID: 6, Elements: makeKVInput(42, test.Input...), Out: precombine}

			constructAndExecutePlan(t, []Unit{n, precombine, gbk, merge, extract, out})
			expected := makeKV(42, test.Expected)
			if !equalList(out.Elements, expected) {
				t.Errorf("liftedCombineChain(%s) = %v, want %v", edge.CombineFn.Name(), extractKeyedValues(out.Elements...), extractKeyedValues(expected...))
			}
		})
	}
}

func getCombineEdge(t *testing.T, cfn interface{}, ac *coder.Coder) *graph.MultiEdge {
	t.Helper()
	fn, err := graph.NewCombineFn(cfn)
	if err != nil {
		t.Fatalf("invalid function: %v", err)
	}

	g := graph.New()
	var vtype reflect.Type
	if fn.AddInputFn() != nil {
		// This makes the assumption that the AddInput function is unkeyed.
		vtype = fn.AddInputFn().Param[1].T
	} else {
		vtype = fn.MergeAccumulatorsFn().Param[0].T
	}
	inT := typex.NewCoGBK(typex.New(reflectx.Int), typex.New(vtype))
	in := g.NewNode(inT, window.DefaultWindowingStrategy(), true)

	edge, err := graph.NewCombine(g, g.Root(), fn, in, ac)
	if err != nil {
		t.Fatalf("invalid combinefn: %v", err)
	}
	return edge
}

func constructAndExecutePlan(t *testing.T, us []Unit) {
	p, err := NewPlan("a", us)
	if err != nil {
		t.Fatalf("failed to construct plan: %v", err)
	}

	if err := p.Execute(context.Background(), "1", DataContext{}); err != nil {
		t.Fatalf("execute failed: %v", err)
	}
	if err := p.Down(context.Background()); err != nil {
		t.Fatalf("down failed: %v", err)
	}
}

// The following functions and struct represent the combine contract implemented
// in various ways.
//
// Instead of a ProcessElement method, a liftable combine can be written by
// breaking down an operation into four component methods:
//
//   CreateAccumulator() AccumT
//   AddInput(a AccumT, v InputT) AccumT
//   MergeAccumulators(a, b AccumT) AccumT
//   ExtractOutput(v AccumT) OutputT
//
// In addition, depending there can be three distinct types, depending on where
// they are used in the combine, the input type, InputT, the output type, OutputT,
// and the accumulator type AccumT. Depending on the equality of the types, one
// or more of the methods can be unspecified.
//
// The only required method is MergeAccumulators.
//
// When InputT == OutputT == AccumT, the only required method is MergeAccumulators.
//
// When InputT == AccumT , then AddInput can be omitted, and MergeAccumulators used instead.
// When AccumT == Output , then ExtractOutput can be omitted, and the identity used instead.
//
// These two combined mean that when InputT == OutputT == AccumT, only MergeAccumulators
// needs to be specified.
//
// CreateAccumulator() is only required if the AccumT cannot be used in it's zero state,
// and needs initialization.

// mergeFn represents a combine that is just a binary merge, where
//
//  InputT == OutputT == AccumT == int
func mergeFn(a, b int) int {
	return a + b
}

// MyCombine represents a combine with the same Input and Output type (int), but a
// distinct accumulator type (int64).
//
//  InputT == OutputT == int
//  AccumT == int64
type MyCombine struct{}

func (*MyCombine) AddInput(a int64, v int) int64 {
	return a + int64(v)
}

func (*MyCombine) MergeAccumulators(a, b int64) int64 {
	return a + b
}

func (*MyCombine) ExtractOutput(a int64) int {
	return int(a)
}

// MyOtherCombine is the same as MyCombine, but has strings extracted as output.
//
//  InputT == int
//  AccumT == int64
//  OutputT == string
type MyOtherCombine struct {
	MyCombine // Embedding to re-use the exisitng AddInput and MergeAccumulators implementations
}

func (*MyOtherCombine) ExtractOutput(a int64) string {
	return fmt.Sprintf("%d", a)
}

// MyThirdCombine has parses strings as Input, and doesn't specify an ExtractOutput
//
//  InputT == string
//  AccumT == int
//  OutputT == int
type MyThirdCombine struct{}

func (c *MyThirdCombine) AddInput(a int, s string) (int, error) {
	v, err := strconv.ParseInt(s, 0, 0)
	if err != nil {
		return 0, err
	}
	return c.MergeAccumulators(a, int(v)), nil
}

func (*MyThirdCombine) MergeAccumulators(a, b int) int {
	return a + b
}

// MyContextCombine is the same as MyCombine, but requires a context parameter.
//
//  InputT == int
//  AccumT == int64
//  OutputT == string
type MyContextCombine struct {
	MyCombine // Embedding to re-use the exisitng AddInput implementations
}

func (*MyContextCombine) MergeAccumulators(_ context.Context, a, b int64) int64 {
	return a + b
}

// MyErrorCombine is the same as MyCombine, but may return an error.
//
//  InputT == int
//  AccumT == int64
//  OutputT == string
type MyErrorCombine struct {
	MyCombine // Embedding to re-use the exisitng AddInput implementations
}

func (*MyErrorCombine) MergeAccumulators(a, b int64) (int64, error) {
	return a + b, nil
}

func intCoder(t reflect.Type) *coder.Coder {
	c, err := coderx.NewVarIntZ(t)
	if err != nil {
		panic(fmt.Sprintf("Couldn't get VarInt coder for %v: %v", t, err))
	}
	return &coder.Coder{Kind: coder.Custom, T: typex.New(t), Custom: c}
}

// simpleGBK buffers all input and continues on FinishBundle. Use with small single-bundle data only.
type simpleGBK struct {
	UID  UnitID
	Edge *graph.MultiEdge
	Out  Node

	m map[interface{}]*group
}

type group struct {
	key    FullValue
	values []FullValue
}

func (n *simpleGBK) ID() UnitID {
	return n.UID
}

func (n *simpleGBK) Up(ctx context.Context) error {
	n.m = make(map[interface{}]*group)
	return nil
}

func (n *simpleGBK) StartBundle(ctx context.Context, id string, data DataContext) error {
	return n.Out.StartBundle(ctx, id, data)
}

func (n *simpleGBK) ProcessElement(ctx context.Context, elm FullValue, _ ...ReStream) error {
	key := elm.Elm.(int)
	value := elm.Elm2

	g, ok := n.m[key]
	if !ok {
		g = &group{
			key:    FullValue{Elm: key, Timestamp: elm.Timestamp, Windows: elm.Windows},
			values: make([]FullValue, 0),
		}
		n.m[key] = g
	}
	g.values = append(g.values, FullValue{Elm: value, Timestamp: elm.Timestamp})

	return nil
}

func (n *simpleGBK) FinishBundle(ctx context.Context) error {
	for _, g := range n.m {
		values := &FixedReStream{Buf: g.values}
		if err := n.Out.ProcessElement(ctx, g.key, values); err != nil {
			return err
		}
	}
	return n.Out.FinishBundle(ctx)
}

func (n *simpleGBK) Down(ctx context.Context) error {
	return nil
}

func (n *simpleGBK) String() string {
	return fmt.Sprintf("simpleGBK: %v Out:%v", n.ID(), n.Out.ID())
}
<commit_msg>Add test for bare nonBinaryMergeFn<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"context"
	"fmt"
	"reflect"
	"runtime"
	"strconv"
	"testing"

	"github.com/apache/beam/sdks/go/pkg/beam/core/graph"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/coderx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
)

var intInput = []interface{}{int(1), int(2), int(3), int(4), int(5), int(6)}
var strInput = []interface{}{"1", "2", "3", "4", "5", "6"}

var tests = []struct {
	Fn         interface{}
	AccumCoder *coder.Coder
	Input      []interface{}
	Expected   interface{}
}{
	{Fn: mergeFn, AccumCoder: intCoder(reflectx.Int), Input: intInput, Expected: int(21)},
	{Fn: nonBinaryMergeFn, AccumCoder: intCoder(reflectx.Int), Input: intInput, Expected: int(21)},
	{Fn: &MyCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: int(21)},
	{Fn: &MyOtherCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: "21"},
	{Fn: &MyThirdCombine{}, AccumCoder: intCoder(reflectx.Int), Input: strInput, Expected: int(21)},
	{Fn: &MyContextCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: int(21)},
	{Fn: &MyErrorCombine{}, AccumCoder: intCoder(reflectx.Int64), Input: intInput, Expected: int(21)},
}

func fnName(x interface{}) string {
	v := reflect.ValueOf(x)
	if v.Kind() != reflect.Func {
		return v.Type().String()
	}
	return runtime.FuncForPC(uintptr(v.Pointer())).Name()
}

// TestCombine verifies that the Combine node works correctly.
func TestCombine(t *testing.T) {
	for _, test := range tests {
		t.Run(fnName(test.Fn), func(t *testing.T) {
			edge := getCombineEdge(t, test.Fn, test.AccumCoder)

			out := &CaptureNode{UID: 1}
			combine := &Combine{UID: 2, Fn: edge.CombineFn, Out: out}
			n := &FixedRoot{UID: 3, Elements: makeKeyedInput(42, test.Input...), Out: combine}

			constructAndExecutePlan(t, []Unit{n, combine, out})

			expected := makeKV(42, test.Expected)
			if !equalList(out.Elements, expected) {
				t.Errorf("combine(%s) = %v, want %v", edge.CombineFn.Name(), extractKeyedValues(out.Elements...), extractKeyedValues(expected...))
			}
		})
	}
}

// TestLiftedCombine verifies that the LiftedCombine, MergeAccumulators, and
// ExtractOutput nodes work correctly after the lift has been performed.
func TestLiftedCombine(t *testing.T) {
	for _, test := range tests {
		t.Run(fnName(test.Fn), func(t *testing.T) {
			edge := getCombineEdge(t, test.Fn, test.AccumCoder)

			out := &CaptureNode{UID: 1}
			extract := &ExtractOutput{Combine: &Combine{UID: 2, Fn: edge.CombineFn, Out: out}}
			merge := &MergeAccumulators{Combine: &Combine{UID: 3, Fn: edge.CombineFn, Out: extract}}
			gbk := &simpleGBK{UID: 4, Out: merge}
			precombine := &LiftedCombine{Combine: &Combine{UID: 5, Fn: edge.CombineFn, Out: gbk}}
			n := &FixedRoot{UID: 6, Elements: makeKVInput(42, test.Input...), Out: precombine}

			constructAndExecutePlan(t, []Unit{n, precombine, gbk, merge, extract, out})
			expected := makeKV(42, test.Expected)
			if !equalList(out.Elements, expected) {
				t.Errorf("liftedCombineChain(%s) = %v, want %v", edge.CombineFn.Name(), extractKeyedValues(out.Elements...), extractKeyedValues(expected...))
			}
		})
	}
}

func getCombineEdge(t *testing.T, cfn interface{}, ac *coder.Coder) *graph.MultiEdge {
	t.Helper()
	fn, err := graph.NewCombineFn(cfn)
	if err != nil {
		t.Fatalf("invalid function: %v", err)
	}

	g := graph.New()
	var vtype reflect.Type
	if fn.AddInputFn() != nil {
		// This makes the assumption that the AddInput function is unkeyed.
		vtype = fn.AddInputFn().Param[1].T
	} else {
		vtype = fn.MergeAccumulatorsFn().Param[1].T
	}
	inT := typex.NewCoGBK(typex.New(reflectx.Int), typex.New(vtype))
	in := g.NewNode(inT, window.DefaultWindowingStrategy(), true)

	edge, err := graph.NewCombine(g, g.Root(), fn, in, ac)
	if err != nil {
		t.Fatalf("invalid combinefn: %v", err)
	}
	return edge
}

func constructAndExecutePlan(t *testing.T, us []Unit) {
	p, err := NewPlan("a", us)
	if err != nil {
		t.Fatalf("failed to construct plan: %v", err)
	}

	if err := p.Execute(context.Background(), "1", DataContext{}); err != nil {
		t.Fatalf("execute failed: %v", err)
	}
	if err := p.Down(context.Background()); err != nil {
		t.Fatalf("down failed: %v", err)
	}
}

// The following functions and struct represent the combine contract implemented
// in various ways.
//
// Instead of a ProcessElement method, a liftable combine can be written by
// breaking down an operation into four component methods:
//
//   CreateAccumulator() AccumT
//   AddInput(a AccumT, v InputT) AccumT
//   MergeAccumulators(a, b AccumT) AccumT
//   ExtractOutput(v AccumT) OutputT
//
// In addition, depending there can be three distinct types, depending on where
// they are used in the combine, the input type, InputT, the output type, OutputT,
// and the accumulator type AccumT. Depending on the equality of the types, one
// or more of the methods can be unspecified.
//
// The only required method is MergeAccumulators.
//
// When InputT == OutputT == AccumT, the only required method is MergeAccumulators.
//
// When InputT == AccumT , then AddInput can be omitted, and MergeAccumulators used instead.
// When AccumT == Output , then ExtractOutput can be omitted, and the identity used instead.
//
// These two combined mean that when InputT == OutputT == AccumT, only MergeAccumulators
// needs to be specified.
//
// CreateAccumulator() is only required if the AccumT cannot be used in it's zero state,
// and needs initialization.

// mergeFn represents a combine that is just a binary merge, where
//
//  InputT == OutputT == AccumT == int
func mergeFn(a, b int) int {
	return a + b
}

// nonBinaryMergeFn represents a combine with a context parameter and an error return, where
//
//  InputT == OutputT == AccumT == int
func nonBinaryMergeFn(ctx context.Context, a, b int) (int, error) {
	return a + b, nil
}

// MyCombine represents a combine with the same Input and Output type (int), but a
// distinct accumulator type (int64).
//
//  InputT == OutputT == int
//  AccumT == int64
type MyCombine struct{}

func (*MyCombine) AddInput(a int64, v int) int64 {
	return a + int64(v)
}

func (*MyCombine) MergeAccumulators(a, b int64) int64 {
	return a + b
}

func (*MyCombine) ExtractOutput(a int64) int {
	return int(a)
}

// MyOtherCombine is the same as MyCombine, but has strings extracted as output.
//
//  InputT == int
//  AccumT == int64
//  OutputT == string
type MyOtherCombine struct {
	MyCombine // Embedding to re-use the exisitng AddInput and MergeAccumulators implementations
}

func (*MyOtherCombine) ExtractOutput(a int64) string {
	return fmt.Sprintf("%d", a)
}

// MyThirdCombine has parses strings as Input, and doesn't specify an ExtractOutput
//
//  InputT == string
//  AccumT == int
//  OutputT == int
type MyThirdCombine struct{}

func (c *MyThirdCombine) AddInput(a int, s string) (int, error) {
	v, err := strconv.ParseInt(s, 0, 0)
	if err != nil {
		return 0, err
	}
	return c.MergeAccumulators(a, int(v)), nil
}

func (*MyThirdCombine) MergeAccumulators(a, b int) int {
	return a + b
}

// MyContextCombine is the same as MyCombine, but requires a context parameter.
//
//  InputT == int
//  AccumT == int64
//  OutputT == string
type MyContextCombine struct {
	MyCombine // Embedding to re-use the exisitng AddInput implementations
}

func (*MyContextCombine) MergeAccumulators(_ context.Context, a, b int64) int64 {
	return a + b
}

// MyErrorCombine is the same as MyCombine, but may return an error.
//
//  InputT == int
//  AccumT == int64
//  OutputT == string
type MyErrorCombine struct {
	MyCombine // Embedding to re-use the exisitng AddInput implementations
}

func (*MyErrorCombine) MergeAccumulators(a, b int64) (int64, error) {
	return a + b, nil
}

func intCoder(t reflect.Type) *coder.Coder {
	c, err := coderx.NewVarIntZ(t)
	if err != nil {
		panic(fmt.Sprintf("Couldn't get VarInt coder for %v: %v", t, err))
	}
	return &coder.Coder{Kind: coder.Custom, T: typex.New(t), Custom: c}
}

// simpleGBK buffers all input and continues on FinishBundle. Use with small single-bundle data only.
type simpleGBK struct {
	UID  UnitID
	Edge *graph.MultiEdge
	Out  Node

	m map[interface{}]*group
}

type group struct {
	key    FullValue
	values []FullValue
}

func (n *simpleGBK) ID() UnitID {
	return n.UID
}

func (n *simpleGBK) Up(ctx context.Context) error {
	n.m = make(map[interface{}]*group)
	return nil
}

func (n *simpleGBK) StartBundle(ctx context.Context, id string, data DataContext) error {
	return n.Out.StartBundle(ctx, id, data)
}

func (n *simpleGBK) ProcessElement(ctx context.Context, elm FullValue, _ ...ReStream) error {
	key := elm.Elm.(int)
	value := elm.Elm2

	g, ok := n.m[key]
	if !ok {
		g = &group{
			key:    FullValue{Elm: key, Timestamp: elm.Timestamp, Windows: elm.Windows},
			values: make([]FullValue, 0),
		}
		n.m[key] = g
	}
	g.values = append(g.values, FullValue{Elm: value, Timestamp: elm.Timestamp})

	return nil
}

func (n *simpleGBK) FinishBundle(ctx context.Context) error {
	for _, g := range n.m {
		values := &FixedReStream{Buf: g.values}
		if err := n.Out.ProcessElement(ctx, g.key, values); err != nil {
			return err
		}
	}
	return n.Out.FinishBundle(ctx)
}

func (n *simpleGBK) Down(ctx context.Context) error {
	return nil
}

func (n *simpleGBK) String() string {
	return fmt.Sprintf("simpleGBK: %v Out:%v", n.ID(), n.Out.ID())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"regexp"
	"strings"

	"github.com/motemen/ghq/utils"
)

// A RemoteRepository represents a remote repository.
type RemoteRepository interface {
	// The repository URL.
	URL() *url.URL
	// Checks if the URL is valid.
	IsValid() bool
	// The VCS backend that hosts the repository.
	VCS() *VCSBackend
}

// A GitHubRepository represents a GitHub repository. Impliments RemoteRepository.
type GitHubRepository struct {
	url *url.URL
}

func (repo *GitHubRepository) URL() *url.URL {
	return repo.url
}

func (repo *GitHubRepository) IsValid() bool {
	if strings.HasPrefix(repo.url.Path, "/blog/") {
		return false
	}

	// must be /{user}/{project}/?
	pathComponents := strings.Split(strings.TrimRight(repo.url.Path, "/"), "/")
	if len(pathComponents) != 3 {
		return false
	}

	return true
}

func (repo *GitHubRepository) VCS() *VCSBackend {
	return GitBackend
}

type GoogleCodeRepository struct {
	url *url.URL
}

func (repo *GoogleCodeRepository) URL() *url.URL {
	return repo.url
}

var validGoogleCodePathPattern = regexp.MustCompile(`^/p/[^/]+/?$`)

func (repo *GoogleCodeRepository) IsValid() bool {
	return validGoogleCodePathPattern.MatchString(repo.url.Path)
}

func (repo *GoogleCodeRepository) VCS() *VCSBackend {
	if utils.RunSilently("hg", "identify", repo.url.String()) == nil {
		return MercurialBackend
	} else if utils.RunSilently("git", "ls-remote", repo.url.String()) == nil {
		return GitBackend
	} else {
		return nil
	}
}

func NewRemoteRepository(url *url.URL) (RemoteRepository, error) {
	if url.Host == "github.com" {
		return &GitHubRepository{url}, nil
	}

	if url.Host == "code.google.com" {
		return &GoogleCodeRepository{url}, nil
	}

	gheHost, err := GitConfig("ghq.ghe.host")

	if err != nil {
		return nil, fmt.Errorf("failed to retrieve GH:E hostname from .gitconfig: %s", err)
	}

	if url.Host == gheHost {
		return &GitHubRepository{url}, nil
	}

	return nil, fmt.Errorf("unsupported host: %s", url.Host)
}
<commit_msg>The config in `.gitconfig` can be multiple to one key.<commit_after>package main

import (
	"fmt"
	"net/url"
	"regexp"
	"strings"

	"github.com/motemen/ghq/utils"
)

// A RemoteRepository represents a remote repository.
type RemoteRepository interface {
	// The repository URL.
	URL() *url.URL
	// Checks if the URL is valid.
	IsValid() bool
	// The VCS backend that hosts the repository.
	VCS() *VCSBackend
}

// A GitHubRepository represents a GitHub repository. Impliments RemoteRepository.
type GitHubRepository struct {
	url *url.URL
}

func (repo *GitHubRepository) URL() *url.URL {
	return repo.url
}

func (repo *GitHubRepository) IsValid() bool {
	if strings.HasPrefix(repo.url.Path, "/blog/") {
		return false
	}

	// must be /{user}/{project}/?
	pathComponents := strings.Split(strings.TrimRight(repo.url.Path, "/"), "/")
	if len(pathComponents) != 3 {
		return false
	}

	return true
}

func (repo *GitHubRepository) VCS() *VCSBackend {
	return GitBackend
}

type GoogleCodeRepository struct {
	url *url.URL
}

func (repo *GoogleCodeRepository) URL() *url.URL {
	return repo.url
}

var validGoogleCodePathPattern = regexp.MustCompile(`^/p/[^/]+/?$`)

func (repo *GoogleCodeRepository) IsValid() bool {
	return validGoogleCodePathPattern.MatchString(repo.url.Path)
}

func (repo *GoogleCodeRepository) VCS() *VCSBackend {
	if utils.RunSilently("hg", "identify", repo.url.String()) == nil {
		return MercurialBackend
	} else if utils.RunSilently("git", "ls-remote", repo.url.String()) == nil {
		return GitBackend
	} else {
		return nil
	}
}

func NewRemoteRepository(url *url.URL) (RemoteRepository, error) {
	if url.Host == "github.com" {
		return &GitHubRepository{url}, nil
	}

	if url.Host == "code.google.com" {
		return &GoogleCodeRepository{url}, nil
	}

	gheHosts, err := GitConfigAll("ghq.ghe.host")

	if err != nil {
		return nil, fmt.Errorf("failed to retrieve GH:E hostname from .gitconfig: %s", err)
	}

	for _, host := range gheHosts {
		if url.Host == host {
			return &GitHubRepository{url}, nil
		}
	}

	return nil, fmt.Errorf("unsupported host: %s", url.Host)
}
<|endoftext|>


<commit_before>package data

import (
	"context"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
	"github.com/lomik/graphite-clickhouse/pkg/dry"
	"github.com/lomik/graphite-clickhouse/pkg/reverse"
	"github.com/lomik/graphite-clickhouse/pkg/scope"
	"github.com/lomik/graphite-clickhouse/pkg/where"
	"go.uber.org/zap"
)

// from, until, step, function, table, prewhere, where
// arrayFilter(x->isNotNull(x)) - do not pass nulls to client
// -Resample - group time and values by time intervals and apply aggregation function
// -OrNull - if there aren't points in an interval, null will be returned
// intDiv(Time, x)*x - round Time down to step multiplier
// TODO: support custom aggregating functions
const queryAggregated = `WITH anyResample(%[1]d, %[2]d, %[3]d)(toUInt32(intDiv(Time, %[3]d)*%[3]d), Time) AS mask
SELECT Path,
 arrayFilter(m->m!=0, mask) AS times,
 arrayFilter((v,m)->m!=0, %[4]sResample(%[1]d, %[2]d, %[3]d)(Value, Time), mask) AS values
FROM %[5]s
%[6]s
%[7]s
GROUP BY Path
FORMAT RowBinary`

// table, prewhere, where
const queryUnaggregated = `SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)
FROM %s
%s
%s
GROUP BY Path
FORMAT RowBinary`

// name of external-data table with metrics paths
const extTableName = "metrics_list"

// CHResponse contains the parsed Data and From/Until timestamps
type CHResponse struct {
	Data  *Data
	From  int64
	Until int64
}

// CHResponses is a slice of CHResponse
type CHResponses []CHResponse

// EmptyResponse returns an CHResponses with one element containing emptyData for the following encoding
func EmptyResponse() CHResponses { return CHResponses{{emptyData, 0, 0}} }

type query struct {
	CHResponses
	cStep            *commonStep
	chURL            string
	chDataTimeout    time.Duration
	chConnectTimeout time.Duration
	debugDir         string
	debugExtDataPerm os.FileMode
	lock             sync.RWMutex
}

type conditions struct {
	*TimeFrame
	*Targets
	// aggregated shows is it request with ClickHouse aggregation or not
	aggregated bool
	// step is used in requests for proper until/from calculation. It's max(steps) for non-aggregated
	// requests and LCM(steps) for aggregated requests
	step int64
	// from is aligned to step
	from int64
	// until is aligned to step
	until int64
	// metricUnreversed grouped by step
	steps map[uint32][]string
	// prewhere contains PREWHERE condition
	prewhere string
	// where contains WHERE condition
	where string
	// metricUnreversed grouped by aggregating function
	aggregations map[string][]string
	// External-data bodies grouped by aggregatig function. For non-aggregated requests "" used as a key
	extDataBodies    map[string]*strings.Builder
	metricsRequested []string
	metricsUnreverse []string
	metricsLookup    []string
}

func newQuery(cfg *config.Config, targets int) *query {
	var cStep *commonStep = nil
	if cfg.ClickHouse.InternalAggregation {
		cStep = &commonStep{
			result: 0,
			wg:     sync.WaitGroup{},
			lock:   sync.RWMutex{},
		}

		cStep.addTargets(targets)
	}

	query := &query{
		CHResponses:      make([]CHResponse, 0, targets),
		cStep:            cStep,
		chURL:            cfg.ClickHouse.Url,
		chDataTimeout:    cfg.ClickHouse.DataTimeout.Value(),
		chConnectTimeout: cfg.Carbonlink.ConnectTimeout.Value(),
		debugDir:         cfg.Debug.Directory,
		debugExtDataPerm: cfg.Debug.ExternalDataPerm.Value(),
		lock:             sync.RWMutex{},
	}

	return query
}

func (q *query) appendReply(chr CHResponse) {
	q.lock.Lock()
	q.CHResponses = append(q.CHResponses, chr)
	q.lock.Unlock()
}

func (q *query) getDataPoints(ctx context.Context, cond *conditions) error {
	logger := scope.Logger(ctx)
	var err error

	err = cond.prepareMetricsLists()
	if err != nil {
		return err
	}
	if len(cond.metricsRequested) == 0 {
		return nil
	}

	// carbonlink request
	carbonlinkResponseRead := queryCarbonlink(ctx, carbonlink, cond.metricsUnreverse)

	cond.prepareLookup()
	cond.setStep(q.cStep)
	cond.setFromUntil()
	cond.setPrewhere()
	cond.setWhere()

	queryContext, queryCancel := context.WithCancel(ctx)
	defer queryCancel()
	data := prepareData(queryContext, len(cond.extDataBodies), carbonlinkResponseRead)

	for agg, extTableBody := range cond.extDataBodies {
		extData := q.metricsListExtData(extTableBody)
		query := cond.generateQuery(agg)
		data.wg.Add(1)
		go func() {
			defer data.wg.Done()
			body, err := clickhouse.Reader(
				scope.WithTable(ctx, cond.pointsTable),
				q.chURL,
				query,
				clickhouse.Options{
					Timeout:        q.chDataTimeout,
					ConnectTimeout: q.chConnectTimeout,
				},
				extData,
			)
			if err != nil {
				logger.Error("reader", zap.Error(err))
				queryCancel()
				data.e <- err
			}
			err = data.parseResponse(queryContext, body, cond)
			if err != nil {
				logger.Error("reader", zap.Error(err))
				queryCancel()
				data.e <- err
			}
		}()
	}

	err = data.wait(queryContext)
	if err != nil {
		logger.Error(
			"data_parser", zap.Error(err), zap.Int("read_bytes", data.length),
			zap.String("runtime", data.spent.String()), zap.Duration("runtime_ns", data.spent),
		)
		return err
	}
	logger.Info(
		"data_parse", zap.Int("read_bytes", data.length), zap.Int("read_points", data.Points.Len()),
		zap.String("runtime", data.spent.String()), zap.Duration("runtime_ns", data.spent),
	)

	data.setSteps(cond)
	data.Points.SetAggregations(cond.aggregations)

	// ClickHouse returns sorted and uniq values, when internal aggregation is used
	// But if carbonlink is used, we still need to sort, filter and rollup points
	if !cond.aggregated || carbonlink != nil {
		sortStart := time.Now()
		data.Points.Sort()
		d := time.Since(sortStart)
		logger.Debug("sort", zap.String("runtime", d.String()), zap.Duration("runtime_ns", d))

		data.Points.Uniq()
		rollupStart := time.Now()
		err = cond.rollupRules.RollupPoints(data.Points, cond.From, data.commonStep)
		if err != nil {
			logger.Error("rollup failed", zap.Error(err))
			return err
		}
		rollupTime := time.Since(rollupStart)
		logger.Debug(
			"rollup",
			zap.String("runtime", rollupTime.String()),
			zap.Duration("runtime_ns", rollupTime),
		)
	}

	data.AM = cond.AM

	q.appendReply(CHResponse{
		Data:  data.Data,
		From:  cond.From,
		Until: cond.Until,
	})
	return nil
}

func (q *query) metricsListExtData(body *strings.Builder) *clickhouse.ExternalData {
	extTable := clickhouse.ExternalTable{
		Name: extTableName,
		Columns: []clickhouse.Column{{
			Name: "Path",
			Type: "String",
		}},
		Format: "TSV",
		Data:   []byte(body.String()),
	}

	extData := clickhouse.NewExternalData(extTable)
	extData.SetDebug(q.debugDir, q.debugExtDataPerm)
	return extData
}

func (c *conditions) prepareMetricsLists() error {
	c.metricsRequested = c.AM.Series(c.isReverse)
	c.metricsUnreverse = c.metricsRequested
	c.metricsLookup = c.metricsRequested

	if c.isReverse {
		c.metricsUnreverse = make([]string, len(c.metricsRequested))
		for i := range c.metricsRequested {
			c.metricsUnreverse[i] = reverse.String(c.metricsRequested[i])
		}

		if c.rollupUseReverted {
			c.metricsLookup = c.metricsUnreverse
		}
	}
	if len(c.metricsRequested) != len(c.metricsUnreverse) {
		return fmt.Errorf(
			"number of metrics in queried an unreversed lists from finder aren't the same: %d vs %d",
			len(c.metricsRequested),
			len(c.metricsUnreverse),
		)
	}
	return nil
}

func (c *conditions) prepareLookup() {
	age := uint32(dry.Max(0, time.Now().Unix()-c.From))
	c.aggregations = make(map[string][]string)
	c.extDataBodies = make(map[string]*strings.Builder)
	c.steps = make(map[uint32][]string)
	aggName := ""

	for i := range c.metricsRequested {
		step, agg := c.rollupRules.Lookup(c.metricsLookup[i], age)

		if _, ok := c.steps[step]; !ok {
			c.steps[step] = make([]string, 0)
		}
		// Fill up metric names for steps only for non-aggregated requests.
		// Aggregated use commonStep
		if !c.aggregated {
			c.steps[step] = append(c.steps[step], c.metricsUnreverse[i])
		}

		// Fill up metric names for aggregations
		if mm, ok := c.aggregations[agg.Name()]; ok {
			c.aggregations[agg.Name()] = append(mm, c.metricsUnreverse[i])
		} else {
			c.aggregations[agg.Name()] = []string{c.metricsUnreverse[i]}
		}

		// Build external-data bodies. For non-aggregated requests there is only one request
		if c.aggregated {
			aggName = agg.Name()
		}
		if mm, ok := c.extDataBodies[aggName]; ok {
			mm.WriteString(c.metricsRequested[i] + "\n")
		} else {
			var mm strings.Builder
			c.extDataBodies[aggName] = &mm
			mm.WriteString(c.metricsRequested[i] + "\n")
		}
	}
}

func (c *conditions) setStep(cStep *commonStep) {
	step := int64(0)
	if !c.aggregated {
		// Use max(steps)
		for s := range c.steps {
			step = dry.Max(step, int64(s))
		}
		c.step = step
		return
	}

	// Use LCM(steps)
	// XXX: This could cause problems, when MutliFetchRequest uses different MaxDataPoints,
	// but currently (2021-04-22) it's not possible
	for s := range c.steps {
		step = cStep.calculateUnsafe(step, int64(s))
	}
	cStep.calculate(step)
	step = dry.Max(cStep.getResult(), dry.Ceil(c.Until-c.From, c.MaxDataPoints))
	c.step = dry.CeilToMultiplier(step, cStep.getResult())
	return
}

func (c *conditions) setFromUntil() {
	c.from = dry.CeilToMultiplier(c.From, c.step)
	c.until = dry.FloorToMultiplier(c.Until, c.step) + c.step - 1
}

func (c *conditions) setPrewhere() {
	pw := where.New()
	pw.And(where.DateBetween("Date", c.from, c.until))
	c.prewhere = pw.PreWhereSQL()
}

func (c *conditions) setWhere() {
	wr := where.New()
	wr.And(where.InTable("Path", extTableName))
	wr.And(where.TimestampBetween("Time", c.from, c.until))
	c.where = wr.SQL()
}

func (c *conditions) generateQuery(agg string) string {
	if c.aggregated {
		return c.generateQueryaAggregated(agg)
	}
	return c.generateQueryUnaggregated()
}

func (c *conditions) generateQueryaAggregated(agg string) string {
	return fmt.Sprintf(
		queryAggregated,
		c.from, c.until, c.step, agg,
		c.pointsTable, c.prewhere, c.where,
	)
}

func (c *conditions) generateQueryUnaggregated() string {
	return fmt.Sprintf(queryUnaggregated, c.pointsTable, c.prewhere, c.where)
}
<commit_msg>Optimize prepareMetricsLists: reverse it only once<commit_after>package data

import (
	"context"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
	"github.com/lomik/graphite-clickhouse/pkg/dry"
	"github.com/lomik/graphite-clickhouse/pkg/reverse"
	"github.com/lomik/graphite-clickhouse/pkg/scope"
	"github.com/lomik/graphite-clickhouse/pkg/where"
	"go.uber.org/zap"
)

// from, until, step, function, table, prewhere, where
// arrayFilter(x->isNotNull(x)) - do not pass nulls to client
// -Resample - group time and values by time intervals and apply aggregation function
// -OrNull - if there aren't points in an interval, null will be returned
// intDiv(Time, x)*x - round Time down to step multiplier
// TODO: support custom aggregating functions
const queryAggregated = `WITH anyResample(%[1]d, %[2]d, %[3]d)(toUInt32(intDiv(Time, %[3]d)*%[3]d), Time) AS mask
SELECT Path,
 arrayFilter(m->m!=0, mask) AS times,
 arrayFilter((v,m)->m!=0, %[4]sResample(%[1]d, %[2]d, %[3]d)(Value, Time), mask) AS values
FROM %[5]s
%[6]s
%[7]s
GROUP BY Path
FORMAT RowBinary`

// table, prewhere, where
const queryUnaggregated = `SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)
FROM %s
%s
%s
GROUP BY Path
FORMAT RowBinary`

// name of external-data table with metrics paths
const extTableName = "metrics_list"

// CHResponse contains the parsed Data and From/Until timestamps
type CHResponse struct {
	Data  *Data
	From  int64
	Until int64
}

// CHResponses is a slice of CHResponse
type CHResponses []CHResponse

// EmptyResponse returns an CHResponses with one element containing emptyData for the following encoding
func EmptyResponse() CHResponses { return CHResponses{{emptyData, 0, 0}} }

type query struct {
	CHResponses
	cStep            *commonStep
	chURL            string
	chDataTimeout    time.Duration
	chConnectTimeout time.Duration
	debugDir         string
	debugExtDataPerm os.FileMode
	lock             sync.RWMutex
}

type conditions struct {
	*TimeFrame
	*Targets
	// aggregated shows is it request with ClickHouse aggregation or not
	aggregated bool
	// step is used in requests for proper until/from calculation. It's max(steps) for non-aggregated
	// requests and LCM(steps) for aggregated requests
	step int64
	// from is aligned to step
	from int64
	// until is aligned to step
	until int64
	// metricUnreversed grouped by step
	steps map[uint32][]string
	// prewhere contains PREWHERE condition
	prewhere string
	// where contains WHERE condition
	where string
	// metricUnreversed grouped by aggregating function
	aggregations map[string][]string
	// External-data bodies grouped by aggregatig function. For non-aggregated requests "" used as a key
	extDataBodies    map[string]*strings.Builder
	metricsRequested []string
	metricsUnreverse []string
	metricsLookup    []string
}

func newQuery(cfg *config.Config, targets int) *query {
	var cStep *commonStep = nil
	if cfg.ClickHouse.InternalAggregation {
		cStep = &commonStep{
			result: 0,
			wg:     sync.WaitGroup{},
			lock:   sync.RWMutex{},
		}

		cStep.addTargets(targets)
	}

	query := &query{
		CHResponses:      make([]CHResponse, 0, targets),
		cStep:            cStep,
		chURL:            cfg.ClickHouse.Url,
		chDataTimeout:    cfg.ClickHouse.DataTimeout.Value(),
		chConnectTimeout: cfg.Carbonlink.ConnectTimeout.Value(),
		debugDir:         cfg.Debug.Directory,
		debugExtDataPerm: cfg.Debug.ExternalDataPerm.Value(),
		lock:             sync.RWMutex{},
	}

	return query
}

func (q *query) appendReply(chr CHResponse) {
	q.lock.Lock()
	q.CHResponses = append(q.CHResponses, chr)
	q.lock.Unlock()
}

func (q *query) getDataPoints(ctx context.Context, cond *conditions) error {
	logger := scope.Logger(ctx)
	var err error

	cond.prepareMetricsLists()
	if len(cond.metricsRequested) == 0 {
		return nil
	}

	// carbonlink request
	carbonlinkResponseRead := queryCarbonlink(ctx, carbonlink, cond.metricsUnreverse)

	cond.prepareLookup()
	cond.setStep(q.cStep)
	cond.setFromUntil()
	cond.setPrewhere()
	cond.setWhere()

	queryContext, queryCancel := context.WithCancel(ctx)
	defer queryCancel()
	data := prepareData(queryContext, len(cond.extDataBodies), carbonlinkResponseRead)

	for agg, extTableBody := range cond.extDataBodies {
		extData := q.metricsListExtData(extTableBody)
		query := cond.generateQuery(agg)
		data.wg.Add(1)
		go func() {
			defer data.wg.Done()
			body, err := clickhouse.Reader(
				scope.WithTable(ctx, cond.pointsTable),
				q.chURL,
				query,
				clickhouse.Options{
					Timeout:        q.chDataTimeout,
					ConnectTimeout: q.chConnectTimeout,
				},
				extData,
			)
			if err != nil {
				logger.Error("reader", zap.Error(err))
				queryCancel()
				data.e <- err
			}
			err = data.parseResponse(queryContext, body, cond)
			if err != nil {
				logger.Error("reader", zap.Error(err))
				queryCancel()
				data.e <- err
			}
		}()
	}

	err = data.wait(queryContext)
	if err != nil {
		logger.Error(
			"data_parser", zap.Error(err), zap.Int("read_bytes", data.length),
			zap.String("runtime", data.spent.String()), zap.Duration("runtime_ns", data.spent),
		)
		return err
	}
	logger.Info(
		"data_parse", zap.Int("read_bytes", data.length), zap.Int("read_points", data.Points.Len()),
		zap.String("runtime", data.spent.String()), zap.Duration("runtime_ns", data.spent),
	)

	data.setSteps(cond)
	data.Points.SetAggregations(cond.aggregations)

	// ClickHouse returns sorted and uniq values, when internal aggregation is used
	// But if carbonlink is used, we still need to sort, filter and rollup points
	if !cond.aggregated || carbonlink != nil {
		sortStart := time.Now()
		data.Points.Sort()
		d := time.Since(sortStart)
		logger.Debug("sort", zap.String("runtime", d.String()), zap.Duration("runtime_ns", d))

		data.Points.Uniq()
		rollupStart := time.Now()
		err = cond.rollupRules.RollupPoints(data.Points, cond.From, data.commonStep)
		if err != nil {
			logger.Error("rollup failed", zap.Error(err))
			return err
		}
		rollupTime := time.Since(rollupStart)
		logger.Debug(
			"rollup",
			zap.String("runtime", rollupTime.String()),
			zap.Duration("runtime_ns", rollupTime),
		)
	}

	data.AM = cond.AM

	q.appendReply(CHResponse{
		Data:  data.Data,
		From:  cond.From,
		Until: cond.Until,
	})
	return nil
}

func (q *query) metricsListExtData(body *strings.Builder) *clickhouse.ExternalData {
	extTable := clickhouse.ExternalTable{
		Name: extTableName,
		Columns: []clickhouse.Column{{
			Name: "Path",
			Type: "String",
		}},
		Format: "TSV",
		Data:   []byte(body.String()),
	}

	extData := clickhouse.NewExternalData(extTable)
	extData.SetDebug(q.debugDir, q.debugExtDataPerm)
	return extData
}

func (c *conditions) prepareMetricsLists() {
	c.metricsUnreverse = c.AM.Series(false)
	c.metricsRequested = c.metricsUnreverse

	if c.isReverse {
		c.metricsRequested = make([]string, len(c.metricsRequested))
		for i := range c.metricsRequested {
			c.metricsRequested[i] = reverse.String(c.metricsUnreverse[i])
		}
	}

	c.metricsLookup = c.metricsRequested
	if c.rollupUseReverted {
		c.metricsLookup = c.metricsUnreverse
	}
}

func (c *conditions) prepareLookup() {
	age := uint32(dry.Max(0, time.Now().Unix()-c.From))
	c.aggregations = make(map[string][]string)
	c.extDataBodies = make(map[string]*strings.Builder)
	c.steps = make(map[uint32][]string)
	aggName := ""

	for i := range c.metricsRequested {
		step, agg := c.rollupRules.Lookup(c.metricsLookup[i], age)

		if _, ok := c.steps[step]; !ok {
			c.steps[step] = make([]string, 0)
		}
		// Fill up metric names for steps only for non-aggregated requests.
		// Aggregated use commonStep
		if !c.aggregated {
			c.steps[step] = append(c.steps[step], c.metricsUnreverse[i])
		}

		// Fill up metric names for aggregations
		if mm, ok := c.aggregations[agg.Name()]; ok {
			c.aggregations[agg.Name()] = append(mm, c.metricsUnreverse[i])
		} else {
			c.aggregations[agg.Name()] = []string{c.metricsUnreverse[i]}
		}

		// Build external-data bodies. For non-aggregated requests there is only one request
		if c.aggregated {
			aggName = agg.Name()
		}
		if mm, ok := c.extDataBodies[aggName]; ok {
			mm.WriteString(c.metricsRequested[i] + "\n")
		} else {
			var mm strings.Builder
			c.extDataBodies[aggName] = &mm
			mm.WriteString(c.metricsRequested[i] + "\n")
		}
	}
}

func (c *conditions) setStep(cStep *commonStep) {
	step := int64(0)
	if !c.aggregated {
		// Use max(steps)
		for s := range c.steps {
			step = dry.Max(step, int64(s))
		}
		c.step = step
		return
	}

	// Use LCM(steps)
	// XXX: This could cause problems, when MutliFetchRequest uses different MaxDataPoints,
	// but currently (2021-04-22) it's not possible
	for s := range c.steps {
		step = cStep.calculateUnsafe(step, int64(s))
	}
	cStep.calculate(step)
	step = dry.Max(cStep.getResult(), dry.Ceil(c.Until-c.From, c.MaxDataPoints))
	c.step = dry.CeilToMultiplier(step, cStep.getResult())
	return
}

func (c *conditions) setFromUntil() {
	c.from = dry.CeilToMultiplier(c.From, c.step)
	c.until = dry.FloorToMultiplier(c.Until, c.step) + c.step - 1
}

func (c *conditions) setPrewhere() {
	pw := where.New()
	pw.And(where.DateBetween("Date", c.from, c.until))
	c.prewhere = pw.PreWhereSQL()
}

func (c *conditions) setWhere() {
	wr := where.New()
	wr.And(where.InTable("Path", extTableName))
	wr.And(where.TimestampBetween("Time", c.from, c.until))
	c.where = wr.SQL()
}

func (c *conditions) generateQuery(agg string) string {
	if c.aggregated {
		return c.generateQueryaAggregated(agg)
	}
	return c.generateQueryUnaggregated()
}

func (c *conditions) generateQueryaAggregated(agg string) string {
	return fmt.Sprintf(
		queryAggregated,
		c.from, c.until, c.step, agg,
		c.pointsTable, c.prewhere, c.where,
	)
}

func (c *conditions) generateQueryUnaggregated() string {
	return fmt.Sprintf(queryUnaggregated, c.pointsTable, c.prewhere, c.where)
}
<|endoftext|>


<commit_before>package config

import (
	"path"

	"github.com/octavore/naga/service"
	"github.com/octavore/nagax/config"
)

type Config struct {
	DataDir string `json:"data_dir"` // themes, plugins
}

type Module struct {
	*config.Module
	Config Config
}

func (m *Module) Init(c *service.Config) {
	c.Setup = func() error {
		return m.ReadConfig(&m.Config)
	}
}

func (m *Module) DataPath(p, backup string) string {
	if p == "" {
		p = backup
	}
	return path.Join(m.Config.DataDir, p)
}
<commit_msg>server: Make ./data the default data dir.<commit_after>package config

import (
	"path"

	"github.com/octavore/naga/service"
	"github.com/octavore/nagax/config"
)

const defaultDataDir = "data"

type Config struct {
	DataDir string `json:"data_dir"` // themes, plugins
}

type Module struct {
	*config.Module
	Config Config
}

func (m *Module) Init(c *service.Config) {
	c.Setup = func() error {
		err := m.ReadConfig(&m.Config)
		if err != nil {
			return err
		}
		if m.Config.DataDir == "" {
			m.Config.DataDir = defaultDataDir
		}
		return nil
	}
}

func (m *Module) DataPath(p, backup string) string {
	if p == "" {
		p = backup
	}
	return path.Join(m.Config.DataDir, p)
}
<|endoftext|>