<commit_before>package pixel

import (
	"image/color"
	"sync"

	"github.com/faiface/pixel/pixelgl"
	"github.com/go-gl/glfw/v3.2/glfw"
	"github.com/pkg/errors"
)

// WindowConfig is convenience structure for specifying all possible properties of a window.
// Properties are chosen in such a way, that you usually only need to set a few of them - defaults
// (zeros) should usually be sensible.
//
// Note that you always need to set the width and the height of a window.
type WindowConfig struct {
	// Title at the top of a window.
	Title string

	// Width of a window in pixels.
	Width float64

	// Height of a window in pixels.
	Height float64

	// If set to nil, a window will be windowed. Otherwise it will be fullscreen on the specified monitor.
	Fullscreen *Monitor

	// Whether a window is resizable.
	Resizable bool

	// If set to true, the window will be initially invisible.
	Hidden bool

	// Undecorated window ommits the borders and decorations (close button, etc.).
	Undecorated bool

	// If set to true, a window will not get focused upon showing up.
	Unfocused bool

	// Whether a window is maximized.
	Maximized bool

	// VSync (vertical synchronization) synchronizes window's framerate with the framerate of the monitor.
	VSync bool

	// Number of samples for multi-sample anti-aliasing (edge-smoothing).
	// Usual values are 0, 2, 4, 8 (powers of 2 and not much more than this).
	MSAASamples int
}

// Window is a window handler. Use this type to manipulate a window (input, drawing, ...).
type Window struct {
	window        *glfw.Window
	config        WindowConfig
	contextHolder pixelgl.ContextHolder

	// need to save these to correctly restore a fullscreen window
	restore struct {
		xpos, ypos, width, height int
	}
}

// NewWindow creates a new window with it's properties specified in the provided config.
//
// If window creation fails, an error is returned.
func NewWindow(config WindowConfig) (*Window, error) {
	bool2int := map[bool]int{
		true:  glfw.True,
		false: glfw.False,
	}

	w := &Window{config: config}

	err := pixelgl.DoErr(func() error {
		err := glfw.Init()
		if err != nil {
			return err
		}

		glfw.WindowHint(glfw.ContextVersionMajor, 3)
		glfw.WindowHint(glfw.ContextVersionMinor, 3)
		glfw.WindowHint(glfw.OpenGLProfile, glfw.OpenGLCoreProfile)
		glfw.WindowHint(glfw.OpenGLForwardCompatible, glfw.True)

		glfw.WindowHint(glfw.Resizable, bool2int[config.Resizable])
		glfw.WindowHint(glfw.Visible, bool2int[!config.Hidden])
		glfw.WindowHint(glfw.Decorated, bool2int[!config.Undecorated])
		glfw.WindowHint(glfw.Focused, bool2int[!config.Unfocused])
		glfw.WindowHint(glfw.Maximized, bool2int[config.Maximized])
		glfw.WindowHint(glfw.Samples, config.MSAASamples)

		w.window, err = glfw.CreateWindow(int(config.Width), int(config.Height), config.Title, nil, nil)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrap(err, "creating window failed")
	}

	w.SetFullscreen(config.Fullscreen)

	defaultShader, err := pixelgl.NewShader(w, defaultVertexFormat, defaultUniformFormat, defaultVertexShader, defaultFragmentShader)
	if err != nil {
		w.Delete()
		return nil, errors.Wrap(err, "creating window failed")
	}

	w.contextHolder.Context = pixelgl.Context{}.WithShader(defaultShader)

	return w, nil
}

// Delete destroys a window. The window can't be used any further.
func (w *Window) Delete() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Destroy()
		})
	})
}

// Clear clears the window with a color.
func (w *Window) Clear(c color.Color) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Clear(colorToRGBA(c))
	})
}

// Update swaps buffers and polls events.
func (w *Window) Update() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			if w.config.VSync {
				glfw.SwapInterval(1)
			}
			w.window.SwapBuffers()
			glfw.PollEvents()
		})
	})
}

// SetTitle changes the title of a window.
func (w *Window) SetTitle(title string) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.SetTitle(title)
		})
	})
}

// SetSize resizes a window to the specified size in pixels.
// In case of a fullscreen window, it changes the resolution of that window.
func (w *Window) SetSize(width, height float64) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.SetSize(int(width), int(height))
		})
	})
}

// Size returns the size of the client area of a window (the part you can draw on).
func (w *Window) Size() (width, height float64) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			wi, hi := w.window.GetSize()
			width = float64(wi)
			height = float64(hi)
		})
	})
	return width, height
}

// Show makes a window visible if it was hidden.
func (w *Window) Show() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Show()
		})
	})
}

// Hide hides a window if it was visible.
func (w *Window) Hide() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Hide()
		})
	})
}

// SetFullscreen sets a window fullscreen on a given monitor. If the monitor is nil, the window will be resored to windowed instead.
//
// Note, that there is nothing about the resolution of the fullscreen window. The window is automatically set to the monitor's
// resolution. If you want a different resolution, you need to set it manually with SetSize method.
func (w *Window) SetFullscreen(monitor *Monitor) {
	if w.Monitor() != monitor {
		if monitor == nil {
			w.Do(func(pixelgl.Context) {
				pixelgl.Do(func() {
					w.window.SetMonitor(
						nil,
						w.restore.xpos,
						w.restore.ypos,
						w.restore.width,
						w.restore.height,
						0,
					)
				})
			})
		} else {
			w.Do(func(pixelgl.Context) {
				pixelgl.Do(func() {
					w.restore.xpos, w.restore.ypos = w.window.GetPos()
					w.restore.width, w.restore.height = w.window.GetSize()

					width, height := monitor.Size()
					refreshRate := monitor.RefreshRate()
					w.window.SetMonitor(
						monitor.monitor,
						0,
						0,
						int(width),
						int(height),
						int(refreshRate),
					)
				})
			})
		}
	}
}

// IsFullscreen returns true if the window is in the fullscreen mode.
func (w *Window) IsFullscreen() bool {
	return w.Monitor() != nil
}

// Monitor returns a monitor a fullscreen window is on. If the window is not fullscreen, this function returns nil.
func (w *Window) Monitor() *Monitor {
	var monitor *glfw.Monitor
	w.Do(func(pixelgl.Context) {
		monitor = pixelgl.DoVal(func() interface{} {
			return w.window.GetMonitor()
		}).(*glfw.Monitor)
	})
	if monitor == nil {
		return nil
	}
	return &Monitor{
		monitor: monitor,
	}
}

// Focus brings a window to the front and sets input focus.
func (w *Window) Focus() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Focus()
		})
	})
}

// Focused returns true if a window has input focus.
func (w *Window) Focused() bool {
	var focused bool
	w.Do(func(pixelgl.Context) {
		focused = pixelgl.DoVal(func() interface{} {
			return w.window.GetAttrib(glfw.Focused) == glfw.True
		}).(bool)
	})
	return focused
}

// Maximize puts a windowed window to a maximized state.
func (w *Window) Maximize() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Maximize()
		})
	})
}

// Restore restores a windowed window from a maximized state.
func (w *Window) Restore() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Restore()
		})
	})
}

var currentWindow struct {
	sync.Mutex
	handler *Window
}

// Do makes the context of this window current, if it's not already, and executes sub.
func (w *Window) Do(sub func(pixelgl.Context)) {
	currentWindow.Lock()
	defer currentWindow.Unlock()

	if currentWindow.handler != w {
		pixelgl.Do(func() {
			w.window.MakeContextCurrent()
			pixelgl.Init()
		})
		currentWindow.handler = w
	}

	w.contextHolder.Do(sub)
}

var defaultVertexFormat = pixelgl.VertexFormat{
	{Purpose: pixelgl.Position, Type: pixelgl.Vec2},
	{Purpose: pixelgl.Color, Type: pixelgl.Vec4},
	{Purpose: pixelgl.TexCoord, Type: pixelgl.Vec2},
}

var defaultUniformFormat = pixelgl.UniformFormat{
	"transform": {Purpose: pixelgl.Transform, Type: pixelgl.Mat3},
	"isTexture": {Purpose: pixelgl.IsTexture, Type: pixelgl.Int},
}

var defaultVertexShader = `
#version 330 core

layout (location = 0) in vec2 position;
layout (location = 1) in vec4 color;
layout (location = 2) in vec2 texCoord;

out vec4 Color;
out vec2 TexCoord;

uniform mat3 transform;

void main() {
	gl_Position = vec4((transform * vec3(position.x, position.y, 1.0)).xy, 0.0, 1.0);
	Color = color;
	TexCoord = texCoord;
}
`

var defaultFragmentShader = `
#version 330 core

in vec4 Color;
in vec2 TexCoord;

out vec4 color;

uniform int isTexture;
uniform sampler2D tex;

void main() {
	if (isTexture != 0) {
		color = Color * texture(tex, vec2(TexCoord.x, 1 - TexCoord.y));
	} else {
		color = Color;
	}
}
`
<commit_msg>fix window context holder<commit_after>package pixel

import (
	"image/color"
	"sync"

	"github.com/faiface/pixel/pixelgl"
	"github.com/go-gl/glfw/v3.2/glfw"
	"github.com/pkg/errors"
)

// WindowConfig is convenience structure for specifying all possible properties of a window.
// Properties are chosen in such a way, that you usually only need to set a few of them - defaults
// (zeros) should usually be sensible.
//
// Note that you always need to set the width and the height of a window.
type WindowConfig struct {
	// Title at the top of a window.
	Title string

	// Width of a window in pixels.
	Width float64

	// Height of a window in pixels.
	Height float64

	// If set to nil, a window will be windowed. Otherwise it will be fullscreen on the specified monitor.
	Fullscreen *Monitor

	// Whether a window is resizable.
	Resizable bool

	// If set to true, the window will be initially invisible.
	Hidden bool

	// Undecorated window ommits the borders and decorations (close button, etc.).
	Undecorated bool

	// If set to true, a window will not get focused upon showing up.
	Unfocused bool

	// Whether a window is maximized.
	Maximized bool

	// VSync (vertical synchronization) synchronizes window's framerate with the framerate of the monitor.
	VSync bool

	// Number of samples for multi-sample anti-aliasing (edge-smoothing).
	// Usual values are 0, 2, 4, 8 (powers of 2 and not much more than this).
	MSAASamples int
}

// Window is a window handler. Use this type to manipulate a window (input, drawing, ...).
type Window struct {
	window        *glfw.Window
	config        WindowConfig
	contextHolder pixelgl.ContextHolder
	defaultShader *pixelgl.Shader

	// need to save these to correctly restore a fullscreen window
	restore struct {
		xpos, ypos, width, height int
	}
}

// NewWindow creates a new window with it's properties specified in the provided config.
//
// If window creation fails, an error is returned.
func NewWindow(config WindowConfig) (*Window, error) {
	bool2int := map[bool]int{
		true:  glfw.True,
		false: glfw.False,
	}

	w := &Window{config: config}

	err := pixelgl.DoErr(func() error {
		err := glfw.Init()
		if err != nil {
			return err
		}

		glfw.WindowHint(glfw.ContextVersionMajor, 3)
		glfw.WindowHint(glfw.ContextVersionMinor, 3)
		glfw.WindowHint(glfw.OpenGLProfile, glfw.OpenGLCoreProfile)
		glfw.WindowHint(glfw.OpenGLForwardCompatible, glfw.True)

		glfw.WindowHint(glfw.Resizable, bool2int[config.Resizable])
		glfw.WindowHint(glfw.Visible, bool2int[!config.Hidden])
		glfw.WindowHint(glfw.Decorated, bool2int[!config.Undecorated])
		glfw.WindowHint(glfw.Focused, bool2int[!config.Unfocused])
		glfw.WindowHint(glfw.Maximized, bool2int[config.Maximized])
		glfw.WindowHint(glfw.Samples, config.MSAASamples)

		w.window, err = glfw.CreateWindow(int(config.Width), int(config.Height), config.Title, nil, nil)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrap(err, "creating window failed")
	}

	w.SetFullscreen(config.Fullscreen)

	defaultShader, err := pixelgl.NewShader(&w.contextHolder, defaultVertexFormat, defaultUniformFormat, defaultVertexShader, defaultFragmentShader)
	if err != nil {
		w.Delete()
		return nil, errors.Wrap(err, "creating window failed")
	}

	w.defaultShader = defaultShader

	return w, nil
}

// Delete destroys a window. The window can't be used any further.
func (w *Window) Delete() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Destroy()
		})
	})
}

// Clear clears the window with a color.
func (w *Window) Clear(c color.Color) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Clear(colorToRGBA(c))
	})
}

// Update swaps buffers and polls events.
func (w *Window) Update() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			if w.config.VSync {
				glfw.SwapInterval(1)
			}
			w.window.SwapBuffers()
			glfw.PollEvents()
		})
	})
}

// SetTitle changes the title of a window.
func (w *Window) SetTitle(title string) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.SetTitle(title)
		})
	})
}

// SetSize resizes a window to the specified size in pixels.
// In case of a fullscreen window, it changes the resolution of that window.
func (w *Window) SetSize(width, height float64) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.SetSize(int(width), int(height))
		})
	})
}

// Size returns the size of the client area of a window (the part you can draw on).
func (w *Window) Size() (width, height float64) {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			wi, hi := w.window.GetSize()
			width = float64(wi)
			height = float64(hi)
		})
	})
	return width, height
}

// Show makes a window visible if it was hidden.
func (w *Window) Show() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Show()
		})
	})
}

// Hide hides a window if it was visible.
func (w *Window) Hide() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Hide()
		})
	})
}

// SetFullscreen sets a window fullscreen on a given monitor. If the monitor is nil, the window will be resored to windowed instead.
//
// Note, that there is nothing about the resolution of the fullscreen window. The window is automatically set to the monitor's
// resolution. If you want a different resolution, you need to set it manually with SetSize method.
func (w *Window) SetFullscreen(monitor *Monitor) {
	if w.Monitor() != monitor {
		if monitor == nil {
			w.Do(func(pixelgl.Context) {
				pixelgl.Do(func() {
					w.window.SetMonitor(
						nil,
						w.restore.xpos,
						w.restore.ypos,
						w.restore.width,
						w.restore.height,
						0,
					)
				})
			})
		} else {
			w.Do(func(pixelgl.Context) {
				pixelgl.Do(func() {
					w.restore.xpos, w.restore.ypos = w.window.GetPos()
					w.restore.width, w.restore.height = w.window.GetSize()

					width, height := monitor.Size()
					refreshRate := monitor.RefreshRate()
					w.window.SetMonitor(
						monitor.monitor,
						0,
						0,
						int(width),
						int(height),
						int(refreshRate),
					)
				})
			})
		}
	}
}

// IsFullscreen returns true if the window is in the fullscreen mode.
func (w *Window) IsFullscreen() bool {
	return w.Monitor() != nil
}

// Monitor returns a monitor a fullscreen window is on. If the window is not fullscreen, this function returns nil.
func (w *Window) Monitor() *Monitor {
	var monitor *glfw.Monitor
	w.Do(func(pixelgl.Context) {
		monitor = pixelgl.DoVal(func() interface{} {
			return w.window.GetMonitor()
		}).(*glfw.Monitor)
	})
	if monitor == nil {
		return nil
	}
	return &Monitor{
		monitor: monitor,
	}
}

// Focus brings a window to the front and sets input focus.
func (w *Window) Focus() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Focus()
		})
	})
}

// Focused returns true if a window has input focus.
func (w *Window) Focused() bool {
	var focused bool
	w.Do(func(pixelgl.Context) {
		focused = pixelgl.DoVal(func() interface{} {
			return w.window.GetAttrib(glfw.Focused) == glfw.True
		}).(bool)
	})
	return focused
}

// Maximize puts a windowed window to a maximized state.
func (w *Window) Maximize() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Maximize()
		})
	})
}

// Restore restores a windowed window from a maximized state.
func (w *Window) Restore() {
	w.Do(func(pixelgl.Context) {
		pixelgl.Do(func() {
			w.window.Restore()
		})
	})
}

var currentWindow struct {
	sync.Mutex
	handler *Window
}

// Do makes the context of this window current, if it's not already, and executes sub.
func (w *Window) Do(sub func(pixelgl.Context)) {
	currentWindow.Lock()
	defer currentWindow.Unlock()

	if currentWindow.handler != w {
		pixelgl.Do(func() {
			w.window.MakeContextCurrent()
			pixelgl.Init()
		})
		currentWindow.handler = w
	}

	if w.defaultShader != nil {
		w.defaultShader.Do(sub)
	} else {
		w.contextHolder.Do(sub)
	}
}

var defaultVertexFormat = pixelgl.VertexFormat{
	{Purpose: pixelgl.Position, Type: pixelgl.Vec2},
	{Purpose: pixelgl.Color, Type: pixelgl.Vec4},
	{Purpose: pixelgl.TexCoord, Type: pixelgl.Vec2},
}

var defaultUniformFormat = pixelgl.UniformFormat{
	"transform": {Purpose: pixelgl.Transform, Type: pixelgl.Mat3},
	"isTexture": {Purpose: pixelgl.IsTexture, Type: pixelgl.Int},
}

var defaultVertexShader = `
#version 330 core

layout (location = 0) in vec2 position;
layout (location = 1) in vec4 color;
layout (location = 2) in vec2 texCoord;

out vec4 Color;
out vec2 TexCoord;

uniform mat3 transform;

void main() {
	gl_Position = vec4((transform * vec3(position.x, position.y, 1.0)).xy, 0.0, 1.0);
	Color = color;
	TexCoord = texCoord;
}
`

var defaultFragmentShader = `
#version 330 core

in vec4 Color;
in vec2 TexCoord;

out vec4 color;

uniform int isTexture;
uniform sampler2D tex;

void main() {
	if (isTexture != 0) {
		color = Color * texture(tex, vec2(TexCoord.x, 1 - TexCoord.y));
	} else {
		color = Color;
	}
}
`
<|endoftext|>


<commit_before>package geom

import "math"

// pointInPolygonal determines whether "pt" is
// within any of the polygons in "pg".
// adapted from https://rosettacode.org/wiki/Ray-casting_algorithm#Go.
// In this version of the algorithm, points that lie on the edge of the polygon
// are considered inside.
func pointInPolygonal(pt Point, pg Polygonal) (in bool) {
	for _, poly := range pg.Polygons() {
		for _, ring := range poly {
			if len(ring) < 3 {
				return false
			}
			// check segment between beginning and ending points
			if !ring[len(ring)-1].Equals(ring[0]) {
				if pointOnSegment(pt, ring[len(ring)-1], ring[0]) {
					return true
				}
				if rayIntersectsSegment(pt, ring[len(ring)-1], ring[0]) {
					in = !in
				}
			}
			// check the rest of the segments.
			for i := 1; i < len(ring); i++ {
				if pointOnSegment(pt, ring[i-1], ring[i]) {
					return true
				}
				if rayIntersectsSegment(pt, ring[i-1], ring[i]) {
					in = !in
				}
			}
		}
	}
	return in
}

func rayIntersectsSegment(p, a, b Point) bool {
	if a.Y > b.Y {
		a, b = b, a
	}
	for p.Y == a.Y || p.Y == b.Y {
		p.Y = math.Nextafter(p.Y, math.Inf(1))
	}
	if p.Y < a.Y || p.Y > b.Y {
		return false
	}
	if a.X > b.X {
		if p.X >= a.X {
			return false
		}
		if p.X < b.X {
			return true
		}
	} else {
		if p.X > b.X {
			return false
		}
		if p.X < a.X {
			return true
		}
	}
	return (p.Y-a.Y)/(p.X-a.X) >= (b.Y-a.Y)/(b.X-a.X)
}
<commit_msg>Sped up Within<commit_after>package geom

import "math"

// pointInPolygonal determines whether "pt" is
// within any of the polygons in "pg".
// adapted from https://rosettacode.org/wiki/Ray-casting_algorithm#Go.
// In this version of the algorithm, points that lie on the edge of the polygon
// are considered inside.
func pointInPolygonal(pt Point, pg Polygonal) (in bool) {
	for _, poly := range pg.Polygons() {
		for _, ring := range poly {
			if len(ring) < 3 {
				continue
			}
			b := NewBounds()
			b.extendPoints(ring)
			if !b.Overlaps(NewBoundsPoint(pt)) {
				continue
			}
			// check segment between beginning and ending points
			if !ring[len(ring)-1].Equals(ring[0]) {
				if pointOnSegment(pt, ring[len(ring)-1], ring[0]) {
					return true
				}
				if rayIntersectsSegment(pt, ring[len(ring)-1], ring[0]) {
					in = !in
				}
			}
			// check the rest of the segments.
			for i := 1; i < len(ring); i++ {
				if pointOnSegment(pt, ring[i-1], ring[i]) {
					return true
				}
				if rayIntersectsSegment(pt, ring[i-1], ring[i]) {
					in = !in
				}
			}
		}
	}
	return in
}

func rayIntersectsSegment(p, a, b Point) bool {
	if a.Y > b.Y {
		a, b = b, a
	}
	for p.Y == a.Y || p.Y == b.Y {
		p.Y = math.Nextafter(p.Y, math.Inf(1))
	}
	if p.Y < a.Y || p.Y > b.Y {
		return false
	}
	if a.X > b.X {
		if p.X >= a.X {
			return false
		}
		if p.X < b.X {
			return true
		}
	} else {
		if p.X > b.X {
			return false
		}
		if p.X < a.X {
			return true
		}
	}
	return (p.Y-a.Y)/(p.X-a.X) >= (b.Y-a.Y)/(b.X-a.X)
}
<|endoftext|>


<commit_before>// Copyright 2010 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/http"
	"os"
	"sort"
	"text/tabwriter"
	"time"

	"github.com/GoogleCloudPlatform/gcping/internal/config"
)

type input struct {
	region   string
	endpoint string
}

func (i *input) HTTP() output {
	return i.benchmark(func() error {
		req, _ := http.NewRequest("GET", i.endpoint+"/ping", nil)
		res, err := client.Do(req)
		if err != nil {
			return err
		}
		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("status code: %v", res.StatusCode)
		}
		return nil
	})
}

func (i *input) benchmark(fn func() error) output {
	if verbose {
		fmt.Printf("Pinging %q\n", i.region)
	}

	start := time.Now()
	err := fn()
	duration := time.Since(start)

	o := output{
		region:    i.region,
		durations: []time.Duration{duration},
	}
	if err != nil {
		o.errors++
	}

	if verbose {
		fmt.Printf("Ping to %q completed in %v\n", i.region, duration)
	}

	if csv {
		fmt.Printf("%v,%v,%v,%v\n", i.region, i.endpoint, duration.Nanoseconds(), err != nil)
	}

	return o
}

type output struct {
	region    string
	durations []time.Duration
	errors    int

	med time.Duration // median of durations; calculated on first call to median()
}

func (o *output) median() time.Duration {
	if o.med == 0 {
		// Sort durations and pick the middle one.
		sort.Slice(o.durations, func(i, j int) bool {
			return o.durations[i] < o.durations[j]
		})
		o.med = o.durations[len(o.durations)/2]
	}
	return o.med

}

type worker struct {
	inputs  chan input
	outputs chan output
}

func (w *worker) start() {
	for worker := 0; worker < concurrency; worker++ {
		go func() {
			for m := range w.inputs {
				o := m.HTTP()
				w.outputs <- o
			}
		}()
	}
}

func (w *worker) sortOutput() []output {
	m := make(map[string]output)
	for i := 0; i < w.size(region); i++ {
		o := <-w.outputs

		a := m[o.region]

		a.region = o.region
		a.durations = append(a.durations, o.durations[0])
		a.errors += o.errors

		m[o.region] = a
	}
	all := make([]output, 0, len(m))
	for _, t := range m {
		all = append(all, t)
	}

	// sort all by median duration.
	sort.Slice(all, func(i, j int) bool {
		return all[i].median() < all[j].median()
	})
	return all
}

func (w *worker) reportAll() {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		for r, e := range config.AllEndpoints {
			w.inputs <- input{region: r, endpoint: e.URL}
		}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	tr := tabwriter.NewWriter(os.Stdout, 3, 2, 2, ' ', 0)
	for i, a := range sorted {
		fmt.Fprintf(tr, "%2d.\t[%v]\t%v", i+1, a.region, a.median())
		if a.errors > 0 {
			fmt.Fprintf(tr, "\t(%d errors)", a.errors)
		}
		fmt.Fprintln(tr)
	}
	tr.Flush()
}

func (w *worker) reportCSV() {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		for r, e := range config.AllEndpoints {
			w.inputs <- input{region: r, endpoint: e.URL}
		}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	fmt.Println("region,latency_ns,errors")
	for _, a := range sorted {
		fmt.Printf("%v,%v,%v\n", a.region, a.median().Nanoseconds(), a.errors)
	}
}

func (w *worker) reportTop() {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		for r, e := range config.AllEndpoints {
			w.inputs <- input{region: r, endpoint: e.URL}
		}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	t := sorted[0].region
	if t == "global" {
		t = sorted[1].region
	}
	fmt.Print(t)
	return
}

func (w *worker) reportRegion(region string) {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		e, _ := config.AllEndpoints[region]
		w.inputs <- input{region: region, endpoint: e.URL}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	fmt.Print(sorted[0].median())

}

func (w *worker) size(region string) int {
	if region != "" {
		return number
	}
	return number * len(config.AllEndpoints)
}
<commit_msg>Fix cli endpoint path (#97)<commit_after>// Copyright 2010 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/http"
	"os"
	"sort"
	"text/tabwriter"
	"time"

	"github.com/GoogleCloudPlatform/gcping/internal/config"
)

type input struct {
	region   string
	endpoint string
}

func (i *input) HTTP() output {
	return i.benchmark(func() error {
		req, _ := http.NewRequest("GET", i.endpoint+"/api/ping", nil)
		res, err := client.Do(req)
		if err != nil {
			return err
		}
		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("status code: %v", res.StatusCode)
		}
		return nil
	})
}

func (i *input) benchmark(fn func() error) output {
	if verbose {
		fmt.Printf("Pinging %q\n", i.region)
	}

	start := time.Now()
	err := fn()
	duration := time.Since(start)

	o := output{
		region:    i.region,
		durations: []time.Duration{duration},
	}
	if err != nil {
		o.errors++
	}

	if verbose {
		fmt.Printf("Ping to %q completed in %v\n", i.region, duration)
	}

	if csv {
		fmt.Printf("%v,%v,%v,%v\n", i.region, i.endpoint, duration.Nanoseconds(), err != nil)
	}

	return o
}

type output struct {
	region    string
	durations []time.Duration
	errors    int

	med time.Duration // median of durations; calculated on first call to median()
}

func (o *output) median() time.Duration {
	if o.med == 0 {
		// Sort durations and pick the middle one.
		sort.Slice(o.durations, func(i, j int) bool {
			return o.durations[i] < o.durations[j]
		})
		o.med = o.durations[len(o.durations)/2]
	}
	return o.med

}

type worker struct {
	inputs  chan input
	outputs chan output
}

func (w *worker) start() {
	for worker := 0; worker < concurrency; worker++ {
		go func() {
			for m := range w.inputs {
				o := m.HTTP()
				w.outputs <- o
			}
		}()
	}
}

func (w *worker) sortOutput() []output {
	m := make(map[string]output)
	for i := 0; i < w.size(region); i++ {
		o := <-w.outputs

		a := m[o.region]

		a.region = o.region
		a.durations = append(a.durations, o.durations[0])
		a.errors += o.errors

		m[o.region] = a
	}
	all := make([]output, 0, len(m))
	for _, t := range m {
		all = append(all, t)
	}

	// sort all by median duration.
	sort.Slice(all, func(i, j int) bool {
		return all[i].median() < all[j].median()
	})
	return all
}

func (w *worker) reportAll() {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		for r, e := range config.AllEndpoints {
			w.inputs <- input{region: r, endpoint: e.URL}
		}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	tr := tabwriter.NewWriter(os.Stdout, 3, 2, 2, ' ', 0)
	for i, a := range sorted {
		fmt.Fprintf(tr, "%2d.\t[%v]\t%v", i+1, a.region, a.median())
		if a.errors > 0 {
			fmt.Fprintf(tr, "\t(%d errors)", a.errors)
		}
		fmt.Fprintln(tr)
	}
	tr.Flush()
}

func (w *worker) reportCSV() {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		for r, e := range config.AllEndpoints {
			w.inputs <- input{region: r, endpoint: e.URL}
		}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	fmt.Println("region,latency_ns,errors")
	for _, a := range sorted {
		fmt.Printf("%v,%v,%v\n", a.region, a.median().Nanoseconds(), a.errors)
	}
}

func (w *worker) reportTop() {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		for r, e := range config.AllEndpoints {
			w.inputs <- input{region: r, endpoint: e.URL}
		}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	t := sorted[0].region
	if t == "global" {
		t = sorted[1].region
	}
	fmt.Print(t)
	return
}

func (w *worker) reportRegion(region string) {
	w.inputs = make(chan input, concurrency)
	w.outputs = make(chan output, w.size(region))
	for i := 0; i < number; i++ {
		e, _ := config.AllEndpoints[region]
		w.inputs <- input{region: region, endpoint: e.URL}
	}
	close(w.inputs)

	sorted := w.sortOutput()
	fmt.Print(sorted[0].median())

}

func (w *worker) size(region string) int {
	if region != "" {
		return number
	}
	return number * len(config.AllEndpoints)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"flag"
	"io"
	"os"
	"strings"
	"sync"
	"log"
	. "github.com/tj/go-debug"

	"github.com/spf13/viper"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"
	"github.com/karalabe/bufioprop" //https://groups.google.com/forum/#!topic/golang-nuts/Mwn9buVnLmY
)

type storage_backend interface {
	GetMetadata(filepath string) (os.FileInfo, error)
	Remove(filePath string) error
	Open(filePath string) (io.Reader, error)
	Create(filePath string) (io.Writer, error)
	Lchown(filePath string, uid, gid int) error
	Chmod(filePath string, perm os.FileMode) error
	Mkdir(dirPath string, perm os.FileMode) error
}

func readWorkerConfig() {
	viper.SetConfigName("config")
	viper.AddConfigPath("$HOME/.pdm")
	viper.AddConfigPath(".")

	viper.SetDefault("dir_workers", 2)
	viper.SetDefault("file_workers", 2)

	err := viper.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}
}

const exchange = "tasks"

var debug = Debug("worker")

var data_backends = make(map[string]storage_backend)

type message struct {
	Body       []byte
	RoutingKey string
}

type task struct {
	Action   string   `json:"action"`
	ItemPath []string `json:"item_path"`
}

type session struct {
	*amqp.Connection
	*amqp.Channel
}

func (s session) Close() error {
	if s.Connection == nil {
		return nil
	}
	return s.Connection.Close()
}

func redial(ctx context.Context, url string) chan chan session {
	sessions := make(chan chan session)

	go func() {
		sess := make(chan session)
		defer close(sessions)

		for {
			select {
			case sessions <- sess:
			case <-ctx.Done():
				log.Println("shutting down session factory")
				return
			}

			conn, err := amqp.Dial(url)
			if err != nil {
				log.Fatalf("cannot (re)dial: %v: %q", err, url)
			}

			ch, err := conn.Channel()
			if err != nil {
				log.Fatalf("cannot create channel: %v", err)
			}

			if err := ch.ExchangeDeclare(exchange, "topic", false, true, false, false, nil); err != nil {
				log.Fatalf("cannot declare exchange: %v", err)
			}

			select {
			case sess <- session{conn, ch}:
			case <-ctx.Done():
				log.Println("shutting down new session")
				return
			}
		}
	}()

	return sessions
}

func publish(sessions chan chan session, messages <-chan message) {
	var (
		running bool
		reading = messages
		pending = make(chan message, 1)
		confirm = make(chan amqp.Confirmation, 1)
	)

	for session := range sessions {
		pub := <-session

		// publisher confirms for this channel/connection
		if err := pub.Confirm(false); err != nil {
			log.Printf("publisher confirms not supported")
			close(confirm) // confirms not supported, simulate by always nacking
		} else {
			pub.NotifyPublish(confirm)
		}

		log.Printf("publishing...")

	Publish:
		for {
			var msg message
			select {
			case confirmed := <-confirm:
				if !confirmed.Ack {
					log.Printf("nack message %d, body: %q", confirmed.DeliveryTag, string(msg.Body))
				}
				reading = messages

			case msg = <-pending:
				err := pub.Publish(exchange, msg.RoutingKey, false, false, amqp.Publishing{
					Body: msg.Body,
				})
				// Retry failed delivery on the next session
				if err != nil {
					pending <- msg
					pub.Close()
					break Publish
				}

			case msg, running = <-reading:
				// all messages consumed
				if !running {
					return
				}
				// work on pending delivery until ack'd
				pending <- msg
				reading = nil
			}
		}
	}
}

func subscribe(sessions chan chan session, file_messages chan<- message, folder_messages chan<- message) {

	for session := range sessions {
		sub := <-session

		var wg sync.WaitGroup

		for k := range viper.Get("datasource").(map[string]interface{}) {
			if viper.GetBool(fmt.Sprintf("datasource.%s.write", k)) {
				for k2 := range viper.Get("datasource").(map[string]interface{}) {
					if k2 != k {
						routingKeyFile, routingKeyDir := fmt.Sprintf("file.%s.%s", k2, k), fmt.Sprintf("dir.%s.%s", k2, k)

						queueFile, err := sub.QueueDeclare("", false, true, true, false, nil)
						if err != nil {
							log.Printf("cannot consume from exclusive queue: %q, %v", queueFile, err)
							return
						}

						if err := sub.QueueBind(queueFile.Name, routingKeyFile, exchange, false, nil); err != nil {
							log.Printf("cannot consume without a binding to exchange: %q, %v", exchange, err)
							return
						}

						deliveriesFile, err := sub.Consume(queueFile.Name, "", false, true, false, false, nil)
						if err != nil {
							log.Printf("cannot consume from: %q, %v", queueFile, err)
							return
						}

						queueDir, err := sub.QueueDeclare("", false, true, true, false, nil)
						if err != nil {
							log.Printf("cannot consume from exclusive queue: %q, %v", queueDir, err)
							return
						}

						if err := sub.QueueBind(queueDir.Name, routingKeyDir, exchange, false, nil); err != nil {
							log.Printf("cannot consume without a binding to exchange: %q, %v", exchange, err)
							return
						}

						deliveriesDir, err := sub.Consume(queueDir.Name, "", false, true, false, false, nil)
						if err != nil {
							log.Printf("cannot consume from: %q, %v", queueDir, err)
							return
						}

						wg.Add(2)

						go func() {
							defer wg.Done()
							for msg := range deliveriesFile {
								var new_msg = message{msg.Body, msg.RoutingKey}
								file_messages <- new_msg
								sub.Ack(msg.DeliveryTag, false)
							}
						}()
						go func() {
							defer wg.Done()
							for msg := range deliveriesDir {
								var new_msg = message{msg.Body, msg.RoutingKey}
								folder_messages <- new_msg
								sub.Ack(msg.DeliveryTag, false)
							}
						}()
					}
				}
			}
		}
		wg.Wait()
	}
}

func read(r io.Reader) <-chan message {
	ret_chan := make(chan message)
	go func() {
		defer close(ret_chan)
		scan := bufio.NewScanner(r)
		for scan.Scan() {
			var msg = message{scan.Bytes(), "file.home.home2"}
			ret_chan <- msg
		}
	}()
	return ret_chan
}

func processFilesStream() chan<- message {
	msgs := make(chan message)
	for i := 0; i <= viper.GetInt("file_workers"); i++ {
		go func(i int) {
			for msg := range msgs {
				var cur_task task
				var fromDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[1]]
				var toDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[2]]
				err := json.Unmarshal(msg.Body, &cur_task)
				if err != nil {
					log.Printf("Error parsing message: %s from %s", msg.Body, msg.RoutingKey)
					continue
				}
				processFiles(fromDataStore, toDataStore, cur_task)
			}
		}(i)
	}
	return msgs
}

func processFoldersStream() chan<- message {
	msgs := make(chan message)
	for i := 0; i <= viper.GetInt("folder_workers"); i++ {
		go func(i int) {
			for msg := range msgs {
				var cur_task task
				var fromDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[1]]
				var toDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[2]]
				err := json.Unmarshal(msg.Body, &cur_task)
				if err != nil {
					log.Printf("Error parsing message: %s", msg.Body)
					continue
				}
				processFolder(fromDataStore, toDataStore, cur_task)
			}
		}(i)
	}
	return msgs
}

func processFiles(fromDataStore storage_backend, toDataStore storage_backend, taskStruct task) {
	for _, filepath := range taskStruct.ItemPath {
		sourceFileMeta, err := fromDataStore.GetMetadata(filepath)
		if err != nil {
			log.Print("Error reading file metadata: ", err)
		}

		debug("For file %s got meta %#v", filepath, sourceFileMeta)

		//TODO: check date

		switch mode := sourceFileMeta.Mode(); {
		case mode.IsRegular():
			//TODO: check stripes

			if destFileMeta, err := toDataStore.GetMetadata(filepath); err == nil { // the dest file exists
				if sourceFileMeta.Size() == destFileMeta.Size() &&
					sourceFileMeta.ModTime() == destFileMeta.ModTime() &&
					sourceFileMeta.Mode() == destFileMeta.Mode() {
					debug("File ", filepath, " hasn't been changed")
					return
				}
				err = toDataStore.Remove(filepath)
				if err != nil {
					log.Print("Error removing file %s: %s", filepath, err)
					continue
				}

				// TODO: setstripe
			}

			src, err := fromDataStore.Open(filepath)
			if err != nil {
				log.Printf("Error opening src file %s: %s", filepath, err)
				continue
			}
			dest, err := toDataStore.Create(filepath)
			if err != nil {
				log.Printf("Error opening dst file %s: %s", filepath, err)
				continue
			}
			copiedData, err := bufioprop.Copy(dest, src, 1048559)
			if err != nil {
				log.Printf("Error copying file %s: %s", filepath, err)
				continue
			}

			// toDataStore.Lchown(filepath, mode&os.ModeSetuid, mode&os.ModeSetgid)
			// toDataStore.Chmod(filepath, syscallMode(perm))

			debug("Done copying %s: %d bytes", filepath, copiedData)

			//{"action":"copy", "item_path":["/filelock.py"]}
			// {"action":"copy", "item_path":["/noc-x86_64-Debian-8.ova"]}

            // copied_data = self.bcopy(src, dst, blksize)
            // os.chmod(dst, srcstat.st_mode)
            // os.utime(dst, (srcstat.st_atime, srcstat.st_mtime))

		case mode.IsDir():
			// shouldn't happen
		case mode&os.ModeSymlink != 0:
			fmt.Println("symbolic link")
		case mode&os.ModeNamedPipe != 0:
			fmt.Println("named pipe")
		}
	}
}

func processFolder(fromDataStore storage_backend, toDataStore storage_backend, taskStruct task) {
	debug("Processing folder!")
	dirPath := taskStruct.ItemPath[0]
	sourceDirMeta, err := fromDataStore.GetMetadata(dirPath)
	if err != nil {
		log.Print("Error reading folder metadata or source folder not exists: ", err)
		return
	}

	debug("Processing folder! 1")
	if destDirMeta, err := toDataStore.GetMetadata(dirPath); err == nil { // the dest folder exists
		debug("%v",destDirMeta)
	} else {
		level := len(strings.Split(dirPath, "/"))
		if(level > 1) {
			toDataStore.Mkdir(dirPath, sourceDirMeta.Mode())
		}
	}
	debug("Processing folder! 2")

    // if(sstat.st_mode != dstat.st_mode):
    //     os.chmod(destdir, sstat.st_mode)
    // if((sstat.st_uid != dstat.st_uid) or (sstat.st_gid != dstat.st_gid)):
    //     os.chown(destdir, sstat.st_uid, sstat.st_gid)

    // slayout = lustreapi.getstripe(sourcedir)
    // dlayout = lustreapi.getstripe(destdir)
    // if slayout.isstriped() != dlayout.isstriped() or slayout.stripecount != dlayout.stripecount:
    //     lustreapi.setstripe(destdir, stripecount=slayout.stripecount)
}

func main() {

	ctx, done := context.WithCancel(context.Background())

	isWorkerParam := flag.Bool("worker", false, "Run a worker")
	rabbitmqServerParam := flag.String("rabbitmq", "", "RABBITMQ server connect string")
	flag.Parse()

	if *isWorkerParam {
		readWorkerConfig()

		for k := range viper.Get("datasource").(map[string]interface{}) {
			switch datastore_type := viper.GetString(fmt.Sprintf("datasource.%s.type", k)); datastore_type {
			case "lustre":
				data_backends[k] = LustreDatastore{
					viper.GetString(fmt.Sprintf("datasource.%s.path", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.mount", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.write", k))}
			case "posix":
				data_backends[k] = PosixDatastore{
					viper.GetString(fmt.Sprintf("datasource.%s.path", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.mount", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.write", k))}
			}
		}

		go func() {
			publish(redial(ctx, viper.GetString("rabbitmq.connect_string")), read(os.Stdin))
			done()
		}()

		go func() {
			subscribe(redial(ctx, viper.GetString("rabbitmq.connect_string")), processFilesStream(), processFoldersStream())
			done()
		}()

	} else {
		rabbitmqServer := ""

		if(os.Getenv("PDM_RABBITMQ") != "") {
			rabbitmqServer = os.Getenv("PDM_RABBITMQ")
		} else if(*rabbitmqServerParam != "") {
			rabbitmqServer = *rabbitmqServerParam
		}

		pub_chan := make(chan message)
		defer close(pub_chan)
		debug("Publishing1")

		go func() {
			// var msg = message{[]byte("{\"action\":\"copy\", \"item_path\":[\"/filelock.py\"]}"), "file.home.home2"}
			var msg = message{[]byte("{\"action\":\"copy\", \"item_path\":[\"/gui\"]}"), "dir.home.home2"}
			pub_chan <- msg
			debug("Publishing2")
		}()

		publish(redial(ctx, rabbitmqServer), pub_chan)
		debug("Publiched")
	}

	<-ctx.Done()

}
<commit_msg>Added params parsing for operations<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"log"
	. "github.com/tj/go-debug"

	"github.com/spf13/viper"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"
	"github.com/karalabe/bufioprop" //https://groups.google.com/forum/#!topic/golang-nuts/Mwn9buVnLmY
	"gopkg.in/alecthomas/kingpin.v2"
)

type storage_backend interface {
	GetMetadata(filepath string) (os.FileInfo, error)
	Remove(filePath string) error
	Open(filePath string) (io.Reader, error)
	Create(filePath string) (io.Writer, error)
	Lchown(filePath string, uid, gid int) error
	Chmod(filePath string, perm os.FileMode) error
	Mkdir(dirPath string, perm os.FileMode) error
}

func readWorkerConfig() {
	viper.SetConfigName("config")
	viper.AddConfigPath("$HOME/.pdm")
	viper.AddConfigPath(".")

	viper.SetDefault("dir_workers", 2)
	viper.SetDefault("file_workers", 2)

	err := viper.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}
}

const exchange = "tasks"

var debug = Debug("worker")

var data_backends = make(map[string]storage_backend)

type message struct {
	Body       []byte
	RoutingKey string
}

type task struct {
	Action   string   `json:"action"`
	ItemPath []string `json:"item_path"`
}

type session struct {
	*amqp.Connection
	*amqp.Channel
}

func (s session) Close() error {
	if s.Connection == nil {
		return nil
	}
	return s.Connection.Close()
}

func redial(ctx context.Context, url string) chan chan session {
	sessions := make(chan chan session)

	go func() {
		sess := make(chan session)
		defer close(sessions)

		for {
			select {
			case sessions <- sess:
			case <-ctx.Done():
				log.Println("shutting down session factory")
				return
			}

			conn, err := amqp.Dial(url)
			if err != nil {
				log.Fatalf("cannot (re)dial: %v: %q", err, url)
			}

			ch, err := conn.Channel()
			if err != nil {
				log.Fatalf("cannot create channel: %v", err)
			}

			if err := ch.ExchangeDeclare(exchange, "topic", false, true, false, false, nil); err != nil {
				log.Fatalf("cannot declare exchange: %v", err)
			}

			select {
			case sess <- session{conn, ch}:
			case <-ctx.Done():
				log.Println("shutting down new session")
				return
			}
		}
	}()

	return sessions
}

func publish(sessions chan chan session, messages <-chan message) {
	var (
		running bool
		reading = messages
		pending = make(chan message, 1)
		confirm = make(chan amqp.Confirmation, 1)
	)

	for session := range sessions {
		pub := <-session

		// publisher confirms for this channel/connection
		if err := pub.Confirm(false); err != nil {
			log.Printf("publisher confirms not supported")
			close(confirm) // confirms not supported, simulate by always nacking
		} else {
			pub.NotifyPublish(confirm)
		}

		log.Printf("publishing...")

	Publish:
		for {
			var msg message
			select {
			case confirmed := <-confirm:
				if !confirmed.Ack {
					log.Printf("nack message %d, body: %q", confirmed.DeliveryTag, string(msg.Body))
				}
				reading = messages

			case msg = <-pending:
				err := pub.Publish(exchange, msg.RoutingKey, false, false, amqp.Publishing{
					Body: msg.Body,
				})
				// Retry failed delivery on the next session
				if err != nil {
					pending <- msg
					pub.Close()
					break Publish
				}

			case msg, running = <-reading:
				// all messages consumed
				if !running {
					return
				}
				// work on pending delivery until ack'd
				pending <- msg
				reading = nil
			}
		}
	}
}

func subscribe(sessions chan chan session, file_messages chan<- message, folder_messages chan<- message) {

	for session := range sessions {
		sub := <-session

		var wg sync.WaitGroup

		for k := range viper.Get("datasource").(map[string]interface{}) {
			if viper.GetBool(fmt.Sprintf("datasource.%s.write", k)) {
				for k2 := range viper.Get("datasource").(map[string]interface{}) {
					if k2 != k {
						routingKeyFile, routingKeyDir := fmt.Sprintf("file.%s.%s", k2, k), fmt.Sprintf("dir.%s.%s", k2, k)

						queueFile, err := sub.QueueDeclare("", false, true, true, false, nil)
						if err != nil {
							log.Printf("cannot consume from exclusive queue: %q, %v", queueFile, err)
							return
						}

						if err := sub.QueueBind(queueFile.Name, routingKeyFile, exchange, false, nil); err != nil {
							log.Printf("cannot consume without a binding to exchange: %q, %v", exchange, err)
							return
						}

						deliveriesFile, err := sub.Consume(queueFile.Name, "", false, true, false, false, nil)
						if err != nil {
							log.Printf("cannot consume from: %q, %v", queueFile, err)
							return
						}

						queueDir, err := sub.QueueDeclare("", false, true, true, false, nil)
						if err != nil {
							log.Printf("cannot consume from exclusive queue: %q, %v", queueDir, err)
							return
						}

						if err := sub.QueueBind(queueDir.Name, routingKeyDir, exchange, false, nil); err != nil {
							log.Printf("cannot consume without a binding to exchange: %q, %v", exchange, err)
							return
						}

						deliveriesDir, err := sub.Consume(queueDir.Name, "", false, true, false, false, nil)
						if err != nil {
							log.Printf("cannot consume from: %q, %v", queueDir, err)
							return
						}

						wg.Add(2)

						go func() {
							defer wg.Done()
							for msg := range deliveriesFile {
								var new_msg = message{msg.Body, msg.RoutingKey}
								file_messages <- new_msg
								sub.Ack(msg.DeliveryTag, false)
							}
						}()
						go func() {
							defer wg.Done()
							for msg := range deliveriesDir {
								var new_msg = message{msg.Body, msg.RoutingKey}
								folder_messages <- new_msg
								sub.Ack(msg.DeliveryTag, false)
							}
						}()
					}
				}
			}
		}
		wg.Wait()
	}
}

func read(r io.Reader) <-chan message {
	ret_chan := make(chan message)
	go func() {
		defer close(ret_chan)
		scan := bufio.NewScanner(r)
		for scan.Scan() {
			var msg = message{scan.Bytes(), "file.home.home2"}
			ret_chan <- msg
		}
	}()
	return ret_chan
}

func processFilesStream() chan<- message {
	msgs := make(chan message)
	for i := 0; i <= viper.GetInt("file_workers"); i++ {
		go func(i int) {
			for msg := range msgs {
				var cur_task task
				var fromDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[1]]
				var toDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[2]]
				err := json.Unmarshal(msg.Body, &cur_task)
				if err != nil {
					log.Printf("Error parsing message: %s from %s", msg.Body, msg.RoutingKey)
					continue
				}
				processFiles(fromDataStore, toDataStore, cur_task)
			}
		}(i)
	}
	return msgs
}

func processFoldersStream() chan<- message {
	msgs := make(chan message)
	for i := 0; i <= viper.GetInt("folder_workers"); i++ {
		go func(i int) {
			for msg := range msgs {
				var cur_task task
				var fromDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[1]]
				var toDataStore = data_backends[strings.Split(msg.RoutingKey, ".")[2]]
				err := json.Unmarshal(msg.Body, &cur_task)
				if err != nil {
					log.Printf("Error parsing message: %s", msg.Body)
					continue
				}
				processFolder(fromDataStore, toDataStore, cur_task)
			}
		}(i)
	}
	return msgs
}

func processFiles(fromDataStore storage_backend, toDataStore storage_backend, taskStruct task) {
	for _, filepath := range taskStruct.ItemPath {
		sourceFileMeta, err := fromDataStore.GetMetadata(filepath)
		if err != nil {
			log.Print("Error reading file metadata: ", err)
		}

		debug("For file %s got meta %#v", filepath, sourceFileMeta)

		//TODO: check date

		switch mode := sourceFileMeta.Mode(); {
		case mode.IsRegular():
			//TODO: check stripes

			if destFileMeta, err := toDataStore.GetMetadata(filepath); err == nil { // the dest file exists
				if sourceFileMeta.Size() == destFileMeta.Size() &&
					sourceFileMeta.ModTime() == destFileMeta.ModTime() &&
					sourceFileMeta.Mode() == destFileMeta.Mode() {
					debug("File ", filepath, " hasn't been changed")
					return
				}
				err = toDataStore.Remove(filepath)
				if err != nil {
					log.Print("Error removing file %s: %s", filepath, err)
					continue
				}

				// TODO: setstripe
			}

			src, err := fromDataStore.Open(filepath)
			if err != nil {
				log.Printf("Error opening src file %s: %s", filepath, err)
				continue
			}
			dest, err := toDataStore.Create(filepath)
			if err != nil {
				log.Printf("Error opening dst file %s: %s", filepath, err)
				continue
			}
			copiedData, err := bufioprop.Copy(dest, src, 1048559)
			if err != nil {
				log.Printf("Error copying file %s: %s", filepath, err)
				continue
			}

			// toDataStore.Lchown(filepath, mode&os.ModeSetuid, mode&os.ModeSetgid)
			// toDataStore.Chmod(filepath, syscallMode(perm))

			debug("Done copying %s: %d bytes", filepath, copiedData)

			//{"action":"copy", "item_path":["/filelock.py"]}
			// {"action":"copy", "item_path":["/noc-x86_64-Debian-8.ova"]}

            // copied_data = self.bcopy(src, dst, blksize)
            // os.chmod(dst, srcstat.st_mode)
            // os.utime(dst, (srcstat.st_atime, srcstat.st_mtime))

		case mode.IsDir():
			// shouldn't happen
		case mode&os.ModeSymlink != 0:
			fmt.Println("symbolic link")
		case mode&os.ModeNamedPipe != 0:
			fmt.Println("named pipe")
		}
	}
}

func processFolder(fromDataStore storage_backend, toDataStore storage_backend, taskStruct task) {
	debug("Processing folder!")
	dirPath := taskStruct.ItemPath[0]
	sourceDirMeta, err := fromDataStore.GetMetadata(dirPath)
	if err != nil {
		log.Print("Error reading folder metadata or source folder not exists: ", err)
		return
	}

	debug("Processing folder! 1")
	if destDirMeta, err := toDataStore.GetMetadata(dirPath); err == nil { // the dest folder exists
		debug("%v",destDirMeta)
	} else {
		level := len(strings.Split(dirPath, "/"))
		if(level > 1) {
			toDataStore.Mkdir(dirPath, sourceDirMeta.Mode())
		}
	}
	debug("Processing folder! 2")

    // if(sstat.st_mode != dstat.st_mode):
    //     os.chmod(destdir, sstat.st_mode)
    // if((sstat.st_uid != dstat.st_uid) or (sstat.st_gid != dstat.st_gid)):
    //     os.chown(destdir, sstat.st_uid, sstat.st_gid)

    // slayout = lustreapi.getstripe(sourcedir)
    // dlayout = lustreapi.getstripe(destdir)
    // if slayout.isstriped() != dlayout.isstriped() or slayout.stripecount != dlayout.stripecount:
    //     lustreapi.setstripe(destdir, stripecount=slayout.stripecount)
}

var (
	app      = kingpin.New("pdm", "Parallel data mover.")

	worker     = app.Command("worker", "Run a worker")

	copy        = app.Command("copy", "Copy a folder or a file")
	rabbitmqServerParam = copy.Flag("rabbitmq", "RabbitMQ connect string.").String()
	isFileParam = copy.Flag("file", "Copy a file.").Bool()
	pathParam    = copy.Arg("path", "The path to copy").Required().String()
)

func main() {
	ctx, done := context.WithCancel(context.Background())

	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
	case worker.FullCommand():
		readWorkerConfig()

		for k := range viper.Get("datasource").(map[string]interface{}) {
			switch datastore_type := viper.GetString(fmt.Sprintf("datasource.%s.type", k)); datastore_type {
			case "lustre":
				data_backends[k] = LustreDatastore{
					viper.GetString(fmt.Sprintf("datasource.%s.path", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.mount", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.write", k))}
			case "posix":
				data_backends[k] = PosixDatastore{
					viper.GetString(fmt.Sprintf("datasource.%s.path", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.mount", k)),
					viper.GetBool(fmt.Sprintf("datasource.%s.write", k))}
			}
		}

		go func() {
			publish(redial(ctx, viper.GetString("rabbitmq.connect_string")), read(os.Stdin))
			done()
		}()

		go func() {
			subscribe(redial(ctx, viper.GetString("rabbitmq.connect_string")), processFilesStream(), processFoldersStream())
			done()
		}()

	case copy.FullCommand():

		rabbitmqServer := ""

		if(os.Getenv("PDM_RABBITMQ") != "") {
			rabbitmqServer = os.Getenv("PDM_RABBITMQ")
		} else if(*rabbitmqServerParam != "") {
			rabbitmqServer = *rabbitmqServerParam
		}

		pub_chan := make(chan message)
		defer close(pub_chan)

		go func() {
			publish(redial(ctx, rabbitmqServer), pub_chan)
		}()

		queuePrefix := "dir"
		if *isFileParam {queuePrefix = "file"}

		var msg = message{[]byte("{\"action\":\"copy\", \"item_path\":[\""+*pathParam+"\"]}"), queuePrefix+".home.home2"}
		pub_chan <- msg
		done()
	}

	<-ctx.Done()

}
<|endoftext|>


<commit_before>package gateway

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/sla"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/status
func (this *manServer) statusHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	output := make(map[string]interface{})
	output["options"] = Options
	output["loglevel"] = logLevel.String()
	output["manager"] = manager.Default.Dump()
	b, _ := json.MarshalIndent(output, "", "    ")
	w.Write(b)
}

// GET /v1/clients
func (this *manServer) clientsHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	b, _ := json.Marshal(this.gw.clientStates.Export())
	w.Write(b)
}

// GET /v1/clusters
func (this *manServer) clustersHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	b, _ := json.Marshal(meta.Default.Clusters())
	w.Write(b)
}

// PUT /v1/options/:option/:value
func (this *manServer) setOptionHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	option := params.ByName("option")
	value := params.ByName("value")
	boolVal := value == "true"

	switch option {
	case "debug":
		Options.Debug = boolVal

	case "clients":
		Options.EnableClientStats = boolVal
		this.gw.clientStates.Reset()

	case "nometrics":
		Options.DisableMetrics = boolVal

	case "gzip":
		Options.EnableGzip = boolVal

	case "ratelimit":
		Options.Ratelimit = boolVal

	case "standbysub":
		Options.PermitStandbySub = boolVal

	case "unregroup":
		Options.PermitUnregisteredGroup = boolVal
		manager.Default.AllowSubWithUnregisteredGroup(boolVal)

	case "maxreq":
		Options.MaxRequestPerConn, _ = strconv.Atoi(value)

	case "accesslog":
		if Options.EnableAccessLog != boolVal {
			// on/off switching
			if boolVal {
				this.gw.accessLogger.Start()
			} else {
				this.gw.accessLogger.Stop()
			}
		}
		Options.EnableAccessLog = boolVal

	default:
		log.Warn("invalid option:%s=%s", option, value)

		writeBadRequest(w, "invalid option")
		return
	}

	log.Info("set option:%s to %s, %#v", option, value, Options)

	w.Write(ResponseOk)
}

// PUT /v1/log/:level
func (this *manServer) setlogHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	logLevel = toLogLevel(params.ByName("level"))
	for name, filter := range log.Global {
		log.Info("log[%s] level: %s -> %s", name, filter.Level, logLevel)

		filter.Level = logLevel
	}

	w.Write(ResponseOk)
}

// DELETE /v1/counter/:name
func (this *manServer) resetCounterHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	counterName := params.ByName("name")

	_ = counterName // TODO

	w.Write(ResponseOk)
}

// GET /v1/partitions/:cluster/:appid/:topic/:ver
func (this *manServer) partitionsHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	topic := params.ByName(UrlParamTopic)
	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName(UrlParamAppid)
	appid := r.Header.Get(HttpHeaderAppid)
	pubkey := r.Header.Get(HttpHeaderPubkey)
	ver := params.ByName(UrlParamVersion)
	if !manager.Default.AuthAdmin(appid, pubkey) {
		log.Warn("suspicous partitions call from %s(%s): {cluster:%s app:%s key:%s topic:%s ver:%s}",
			r.RemoteAddr, getHttpRemoteIp(r), cluster, appid, pubkey, topic, ver)

		writeAuthFailure(w, manager.ErrAuthenticationFail)
		return
	}

	zkcluster := meta.Default.ZkCluster(cluster)
	if zkcluster == nil {
		log.Error("suspicous partitions call from %s(%s): {cluster:%s app:%s key:%s topic:%s ver:%s} undefined cluster",
			r.RemoteAddr, getHttpRemoteIp(r), cluster, appid, pubkey, topic, ver)

		writeBadRequest(w, "undefined cluster")
		return
	}

	kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
	if err != nil {
		log.Error("cluster[%s] %v", zkcluster.Name(), err)

		writeServerError(w, err.Error())
		return
	}
	defer kfk.Close()

	partitions, err := kfk.Partitions(manager.Default.KafkaTopic(hisAppid, topic, ver))
	if err != nil {
		log.Error("cluster[%s] from %s(%s) {app:%s topic:%s ver:%s} %v",
			zkcluster.Name(), r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, err)

		writeServerError(w, err.Error())
		return
	}

	w.Write([]byte(fmt.Sprintf(`{"num": %d}`, len(partitions))))
}

// POST /v1/topics/:cluster/:appid/:topic/:ver?partitions=1&replicas=2&retention.hours=72&retention.bytes=-1
func (this *manServer) addTopicHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	topic := params.ByName(UrlParamTopic)
	if !manager.Default.ValidateTopicName(topic) {
		log.Warn("illegal topic: %s", topic)

		writeBadRequest(w, "illegal topic")
		return
	}

	if !this.throttleAddTopic.Pour(getHttpRemoteIp(r), 1) {
		writeQuotaExceeded(w)
		return
	}

	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName(UrlParamAppid)
	appid := r.Header.Get(HttpHeaderAppid)
	pubkey := r.Header.Get(HttpHeaderPubkey)
	ver := params.ByName(UrlParamVersion)
	if !manager.Default.AuthAdmin(appid, pubkey) {
		log.Warn("suspicous add topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s}",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeAuthFailure(w, manager.ErrAuthenticationFail)
		return
	}

	zkcluster := meta.Default.ZkCluster(cluster)
	if zkcluster == nil {
		log.Error("add topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s} undefined cluster",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeBadRequest(w, "undefined cluster")
		return
	}

	info := zkcluster.RegisteredInfo()
	if !info.Public {
		log.Warn("app[%s] adding topic:%s in non-public cluster: %+v", hisAppid, topic, params)

		writeBadRequest(w, "invalid cluster")
		return
	}

	ts := sla.DefaultSla()
	query := r.URL.Query()
	if partitionsArg := query.Get(sla.SlaKeyPartitions); partitionsArg != "" {
		ts.Partitions, _ = strconv.Atoi(partitionsArg)
	}
	if replicasArg := query.Get(sla.SlaKeyReplicas); replicasArg != "" {
		ts.Replicas, _ = strconv.Atoi(replicasArg)
	}
	if retentionBytes := query.Get(sla.SlaKeyRetentionBytes); retentionBytes != "" {
		ts.RetentionBytes, _ = strconv.Atoi(retentionBytes)
	}
	ts.ParseRetentionHours(query.Get(sla.SlaKeyRetentionHours))

	// validate the sla
	if err := ts.Validate(); err != nil {
		log.Error("app[%s] update topic:%s %s: %+v", hisAppid, topic, query.Encode(), err)

		writeBadRequest(w, err.Error())
		return
	}

	log.Info("app[%s] from %s(%s) add topic: {appid:%s cluster:%s topic:%s ver:%s query:%s}",
		appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode())

	topic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	lines, err := zkcluster.AddTopic(topic, ts)
	if err != nil {
		log.Error("app[%s] %s add topic: %s", appid, r.RemoteAddr, err.Error())

		writeServerError(w, err.Error())
		return
	}

	createdOk := false
	for _, l := range lines {
		log.Trace("app[%s] add topic[%s] in cluster %s: %s", appid, topic, cluster, l)

		if strings.Contains(l, "Created topic") {
			createdOk = true
		}
	}

	if createdOk {
		alterConfig := ts.DumpForAlterTopic()
		if len(alterConfig) == 0 {
			w.Write(ResponseOk)
			return
		}

		lines, err = zkcluster.AlterTopic(topic, ts)
		if err != nil {
			log.Error("app[%s] %s alter topic: %s", appid, r.RemoteAddr, err.Error())

			writeServerError(w, err.Error())
			return
		}

		for _, l := range lines {
			log.Trace("app[%s] alter topic[%s] in cluster %s: %s", appid, topic, cluster, l)
		}

		w.Write(ResponseOk)
	} else {
		writeServerError(w, strings.Join(lines, ";"))
	}
}

// PUT /v1/topics/:cluster/:appid/:topic/:ver?partitions=1&retention.hours=72&retention.bytes=-1
func (this *manServer) updateTopicHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	topic := params.ByName(UrlParamTopic)
	if !manager.Default.ValidateTopicName(topic) {
		log.Warn("illegal topic: %s", topic)

		writeBadRequest(w, "illegal topic")
		return
	}

	if !this.throttleAddTopic.Pour(getHttpRemoteIp(r), 1) {
		writeQuotaExceeded(w)
		return
	}

	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName(UrlParamAppid)
	appid := r.Header.Get(HttpHeaderAppid)
	pubkey := r.Header.Get(HttpHeaderPubkey)
	ver := params.ByName(UrlParamVersion)
	if !manager.Default.AuthAdmin(appid, pubkey) {
		log.Warn("suspicous update topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s}",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeAuthFailure(w, manager.ErrAuthenticationFail)
		return
	}

	zkcluster := meta.Default.ZkCluster(cluster)
	if zkcluster == nil {
		log.Error("update topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s} undefined cluster",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeBadRequest(w, "undefined cluster")
		return
	}

	info := zkcluster.RegisteredInfo()
	if !info.Public {
		log.Warn("app[%s] update topic:%s in non-public cluster: %+v", hisAppid, topic, params)

		writeBadRequest(w, "invalid cluster")
		return
	}

	ts := sla.DefaultSla()
	query := r.URL.Query()
	if partitionsArg := query.Get(sla.SlaKeyPartitions); partitionsArg != "" {
		ts.Partitions, _ = strconv.Atoi(partitionsArg)
	}
	if retentionBytes := query.Get(sla.SlaKeyRetentionBytes); retentionBytes != "" {
		ts.RetentionBytes, _ = strconv.Atoi(retentionBytes)
	}
	ts.ParseRetentionHours(query.Get(sla.SlaKeyRetentionHours))

	// validate the sla
	if err := ts.Validate(); err != nil {
		log.Error("app[%s] update topic:%s %s: %+v", hisAppid, topic, query.Encode(), err)

		writeBadRequest(w, err.Error())
		return
	}

	log.Info("app[%s] from %s(%s) update topic: {appid:%s cluster:%s topic:%s ver:%s query:%s}",
		appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode())

	rawTopic := manager.Default.KafkaTopic(hisAppid, topic, ver)
	alterConfig := ts.DumpForAlterTopic()
	if len(alterConfig) == 0 {
		log.Warn("app[%s] from %s(%s) update topic: {appid:%s cluster:%s topic:%s ver:%s query:%s} nothing updated",
			appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode())

		w.Write(ResponseOk)
		return
	}

	lines, err := zkcluster.AlterTopic(rawTopic, ts)
	if err != nil {
		log.Error("app[%s] from %s(%s) update topic: {appid:%s cluster:%s topic:%s ver:%s query:%s} %v",
			appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode(), err)

		writeServerError(w, err.Error())
		return
	}

	for _, l := range lines {
		log.Trace("app[%s] update topic[%s] in cluster %s: %s", appid, rawTopic, cluster, l)
	}

	w.Write(ResponseOk)
}
<commit_msg>updateTopicHandler: when nothing updated, return 400 status code<commit_after>package gateway

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/sla"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/status
func (this *manServer) statusHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	output := make(map[string]interface{})
	output["options"] = Options
	output["loglevel"] = logLevel.String()
	output["manager"] = manager.Default.Dump()
	b, _ := json.MarshalIndent(output, "", "    ")
	w.Write(b)
}

// GET /v1/clients
func (this *manServer) clientsHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	b, _ := json.Marshal(this.gw.clientStates.Export())
	w.Write(b)
}

// GET /v1/clusters
func (this *manServer) clustersHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	b, _ := json.Marshal(meta.Default.Clusters())
	w.Write(b)
}

// PUT /v1/options/:option/:value
func (this *manServer) setOptionHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	option := params.ByName("option")
	value := params.ByName("value")
	boolVal := value == "true"

	switch option {
	case "debug":
		Options.Debug = boolVal

	case "clients":
		Options.EnableClientStats = boolVal
		this.gw.clientStates.Reset()

	case "nometrics":
		Options.DisableMetrics = boolVal

	case "gzip":
		Options.EnableGzip = boolVal

	case "ratelimit":
		Options.Ratelimit = boolVal

	case "standbysub":
		Options.PermitStandbySub = boolVal

	case "unregroup":
		Options.PermitUnregisteredGroup = boolVal
		manager.Default.AllowSubWithUnregisteredGroup(boolVal)

	case "maxreq":
		Options.MaxRequestPerConn, _ = strconv.Atoi(value)

	case "accesslog":
		if Options.EnableAccessLog != boolVal {
			// on/off switching
			if boolVal {
				this.gw.accessLogger.Start()
			} else {
				this.gw.accessLogger.Stop()
			}
		}
		Options.EnableAccessLog = boolVal

	default:
		log.Warn("invalid option:%s=%s", option, value)

		writeBadRequest(w, "invalid option")
		return
	}

	log.Info("set option:%s to %s, %#v", option, value, Options)

	w.Write(ResponseOk)
}

// PUT /v1/log/:level
func (this *manServer) setlogHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	logLevel = toLogLevel(params.ByName("level"))
	for name, filter := range log.Global {
		log.Info("log[%s] level: %s -> %s", name, filter.Level, logLevel)

		filter.Level = logLevel
	}

	w.Write(ResponseOk)
}

// DELETE /v1/counter/:name
func (this *manServer) resetCounterHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	counterName := params.ByName("name")

	_ = counterName // TODO

	w.Write(ResponseOk)
}

// GET /v1/partitions/:cluster/:appid/:topic/:ver
func (this *manServer) partitionsHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	topic := params.ByName(UrlParamTopic)
	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName(UrlParamAppid)
	appid := r.Header.Get(HttpHeaderAppid)
	pubkey := r.Header.Get(HttpHeaderPubkey)
	ver := params.ByName(UrlParamVersion)
	if !manager.Default.AuthAdmin(appid, pubkey) {
		log.Warn("suspicous partitions call from %s(%s): {cluster:%s app:%s key:%s topic:%s ver:%s}",
			r.RemoteAddr, getHttpRemoteIp(r), cluster, appid, pubkey, topic, ver)

		writeAuthFailure(w, manager.ErrAuthenticationFail)
		return
	}

	zkcluster := meta.Default.ZkCluster(cluster)
	if zkcluster == nil {
		log.Error("suspicous partitions call from %s(%s): {cluster:%s app:%s key:%s topic:%s ver:%s} undefined cluster",
			r.RemoteAddr, getHttpRemoteIp(r), cluster, appid, pubkey, topic, ver)

		writeBadRequest(w, "undefined cluster")
		return
	}

	kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
	if err != nil {
		log.Error("cluster[%s] %v", zkcluster.Name(), err)

		writeServerError(w, err.Error())
		return
	}
	defer kfk.Close()

	partitions, err := kfk.Partitions(manager.Default.KafkaTopic(hisAppid, topic, ver))
	if err != nil {
		log.Error("cluster[%s] from %s(%s) {app:%s topic:%s ver:%s} %v",
			zkcluster.Name(), r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, err)

		writeServerError(w, err.Error())
		return
	}

	w.Write([]byte(fmt.Sprintf(`{"num": %d}`, len(partitions))))
}

// POST /v1/topics/:cluster/:appid/:topic/:ver?partitions=1&replicas=2&retention.hours=72&retention.bytes=-1
func (this *manServer) addTopicHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	topic := params.ByName(UrlParamTopic)
	if !manager.Default.ValidateTopicName(topic) {
		log.Warn("illegal topic: %s", topic)

		writeBadRequest(w, "illegal topic")
		return
	}

	if !this.throttleAddTopic.Pour(getHttpRemoteIp(r), 1) {
		writeQuotaExceeded(w)
		return
	}

	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName(UrlParamAppid)
	appid := r.Header.Get(HttpHeaderAppid)
	pubkey := r.Header.Get(HttpHeaderPubkey)
	ver := params.ByName(UrlParamVersion)
	if !manager.Default.AuthAdmin(appid, pubkey) {
		log.Warn("suspicous add topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s}",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeAuthFailure(w, manager.ErrAuthenticationFail)
		return
	}

	zkcluster := meta.Default.ZkCluster(cluster)
	if zkcluster == nil {
		log.Error("add topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s} undefined cluster",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeBadRequest(w, "undefined cluster")
		return
	}

	info := zkcluster.RegisteredInfo()
	if !info.Public {
		log.Warn("app[%s] adding topic:%s in non-public cluster: %+v", hisAppid, topic, params)

		writeBadRequest(w, "invalid cluster")
		return
	}

	ts := sla.DefaultSla()
	query := r.URL.Query()
	if partitionsArg := query.Get(sla.SlaKeyPartitions); partitionsArg != "" {
		ts.Partitions, _ = strconv.Atoi(partitionsArg)
	}
	if replicasArg := query.Get(sla.SlaKeyReplicas); replicasArg != "" {
		ts.Replicas, _ = strconv.Atoi(replicasArg)
	}
	if retentionBytes := query.Get(sla.SlaKeyRetentionBytes); retentionBytes != "" {
		ts.RetentionBytes, _ = strconv.Atoi(retentionBytes)
	}
	ts.ParseRetentionHours(query.Get(sla.SlaKeyRetentionHours))

	// validate the sla
	if err := ts.Validate(); err != nil {
		log.Error("app[%s] update topic:%s %s: %+v", hisAppid, topic, query.Encode(), err)

		writeBadRequest(w, err.Error())
		return
	}

	log.Info("app[%s] from %s(%s) add topic: {appid:%s cluster:%s topic:%s ver:%s query:%s}",
		appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode())

	topic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	lines, err := zkcluster.AddTopic(topic, ts)
	if err != nil {
		log.Error("app[%s] %s add topic: %s", appid, r.RemoteAddr, err.Error())

		writeServerError(w, err.Error())
		return
	}

	createdOk := false
	for _, l := range lines {
		log.Trace("app[%s] add topic[%s] in cluster %s: %s", appid, topic, cluster, l)

		if strings.Contains(l, "Created topic") {
			createdOk = true
		}
	}

	if createdOk {
		alterConfig := ts.DumpForAlterTopic()
		if len(alterConfig) == 0 {
			w.Write(ResponseOk)
			return
		}

		lines, err = zkcluster.AlterTopic(topic, ts)
		if err != nil {
			log.Error("app[%s] %s alter topic: %s", appid, r.RemoteAddr, err.Error())

			writeServerError(w, err.Error())
			return
		}

		for _, l := range lines {
			log.Trace("app[%s] alter topic[%s] in cluster %s: %s", appid, topic, cluster, l)
		}

		w.Write(ResponseOk)
	} else {
		writeServerError(w, strings.Join(lines, ";"))
	}
}

// PUT /v1/topics/:cluster/:appid/:topic/:ver?partitions=1&retention.hours=72&retention.bytes=-1
func (this *manServer) updateTopicHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	topic := params.ByName(UrlParamTopic)
	if !manager.Default.ValidateTopicName(topic) {
		log.Warn("illegal topic: %s", topic)

		writeBadRequest(w, "illegal topic")
		return
	}

	if !this.throttleAddTopic.Pour(getHttpRemoteIp(r), 1) {
		writeQuotaExceeded(w)
		return
	}

	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName(UrlParamAppid)
	appid := r.Header.Get(HttpHeaderAppid)
	pubkey := r.Header.Get(HttpHeaderPubkey)
	ver := params.ByName(UrlParamVersion)
	if !manager.Default.AuthAdmin(appid, pubkey) {
		log.Warn("suspicous update topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s}",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeAuthFailure(w, manager.ErrAuthenticationFail)
		return
	}

	zkcluster := meta.Default.ZkCluster(cluster)
	if zkcluster == nil {
		log.Error("update topic from %s(%s): {appid:%s pubkey:%s cluster:%s topic:%s ver:%s} undefined cluster",
			r.RemoteAddr, getHttpRemoteIp(r), appid, pubkey, cluster, topic, ver)

		writeBadRequest(w, "undefined cluster")
		return
	}

	info := zkcluster.RegisteredInfo()
	if !info.Public {
		log.Warn("app[%s] update topic:%s in non-public cluster: %+v", hisAppid, topic, params)

		writeBadRequest(w, "invalid cluster")
		return
	}

	ts := sla.DefaultSla()
	query := r.URL.Query()
	if partitionsArg := query.Get(sla.SlaKeyPartitions); partitionsArg != "" {
		ts.Partitions, _ = strconv.Atoi(partitionsArg)
	}
	if retentionBytes := query.Get(sla.SlaKeyRetentionBytes); retentionBytes != "" {
		ts.RetentionBytes, _ = strconv.Atoi(retentionBytes)
	}
	ts.ParseRetentionHours(query.Get(sla.SlaKeyRetentionHours))

	// validate the sla
	if err := ts.Validate(); err != nil {
		log.Error("app[%s] update topic:%s %s: %+v", hisAppid, topic, query.Encode(), err)

		writeBadRequest(w, err.Error())
		return
	}

	log.Info("app[%s] from %s(%s) update topic: {appid:%s cluster:%s topic:%s ver:%s query:%s}",
		appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode())

	rawTopic := manager.Default.KafkaTopic(hisAppid, topic, ver)
	alterConfig := ts.DumpForAlterTopic()
	if len(alterConfig) == 0 {
		log.Warn("app[%s] from %s(%s) update topic: {appid:%s cluster:%s topic:%s ver:%s query:%s} nothing updated",
			appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode())

		writeBadRequest(w, "nothing updated")
		return
	}

	lines, err := zkcluster.AlterTopic(rawTopic, ts)
	if err != nil {
		log.Error("app[%s] from %s(%s) update topic: {appid:%s cluster:%s topic:%s ver:%s query:%s} %v",
			appid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, cluster, topic, ver, query.Encode(), err)

		writeServerError(w, err.Error())
		return
	}

	for _, l := range lines {
		log.Trace("app[%s] update topic[%s] in cluster %s: %s", appid, rawTopic, cluster, l)
	}

	w.Write(ResponseOk)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/subtle"
	"encoding/json"
	"net/http"
	"time"

	"github.com/Cloud-Foundations/keymaster/lib/instrumentedwriter"
	"github.com/Cloud-Foundations/keymaster/lib/webapi/v0/proto"
)

const bootstrapOtpAuthPath = "/api/v0/bootstrapOtpAuth"

func (state *RuntimeState) BootstrapOtpAuthHandler(w http.ResponseWriter,
	r *http.Request) {
	if state.sendFailureToClientIfLocked(w, r) {
		return
	}
	if r.Method != "POST" {
		state.writeFailureResponse(w, r, http.StatusMethodNotAllowed, "")
		return
	}
	state.logger.Debugf(3, "Got client POST connection")
	if err := r.ParseForm(); err != nil {
		state.logger.Println(err)
		state.writeFailureResponse(w, r, http.StatusBadRequest,
			"Error parsing form")
		return
	}
	authUser, currentAuthLevel, err := state.checkAuth(w, r, AuthTypeAny)
	if err != nil {
		state.logger.Debugf(1, "%v", err)
		return
	}
	w.(*instrumentedwriter.LoggingWriter).SetUsername(authUser)
	var inputOTP string
	if val, ok := r.Form["OTP"]; ok {
		if len(val) > 1 {
			state.writeFailureResponse(w, r, http.StatusBadRequest,
				"Just one OTP Value allowed")
			state.logger.Printf("Login with multiple OTP Values")
			return
		}
		inputOTP = val[0]
	}
	profile, _, fromCache, err := state.LoadUserProfile(authUser)
	if err != nil {
		state.logger.Printf("error loading user profile err=%s", err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError,
			"Failure loading user profile")
		return
	}
	if fromCache {
		state.writeFailureResponse(w, r, http.StatusServiceUnavailable,
			"Working in DB disconnected mode, try again later")
		return
	}
	requiredOTP := state.userBootstrapOtp(profile, fromCache)
	if requiredOTP == "" {
		state.writeFailureResponse(w, r, http.StatusPreconditionFailed,
			"No valid Bootstrap OTP saved")
		return
	}
	if subtle.ConstantTimeCompare([]byte(inputOTP), []byte(requiredOTP)) != 1 {
		state.logger.Debugf(0, "Invalid Bootstrap OTP value for %s\n",
			authUser)
		state.logger.Debugf(4, "  input: \"%s\" required: \"%s\"\n",
			inputOTP, requiredOTP)
		state.writeFailureResponse(w, r, http.StatusUnauthorized,
			"Invalid Bootstrap OTP")
		return
	}
	profile.BootstrapOTP = bootstrapOTPData{}
	if err := state.SaveUserProfile(authUser, profile); err != nil {
		state.logger.Printf("error saving profile randr=%s", err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError, "")
		return
	}
	_, err = state.updateAuthCookieAuthlevel(w, r,
		currentAuthLevel|AuthTypeBootstrapOTP)
	if err != nil {
		logger.Printf("Auth Cookie NOT found ? %s", err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError,
			"Failure when validating Boostrap OTP")
		return
	}
	// eventNotifier.PublishBootstrapOtpAuthEvent(eventmon.AuthTypeBootstrapOTP,
	// authUser)
	// Now we send the user to the appropriate place
	returnAcceptType := getPreferredAcceptType(r)
	// TODO: The cert backend should depend also on per user preferences.
	loginResponse := proto.LoginResponse{Message: "success"}
	switch returnAcceptType {
	case "text/html":
		loginDestination := getLoginDestination(r)
		eventNotifier.PublishWebLoginEvent(authUser)
		state.logger.Debugf(0, "redirecting to: %s\n", loginDestination)
		http.Redirect(w, r, loginDestination, 302)
	default:
		w.WriteHeader(http.StatusOK)
		json.NewEncoder(w).Encode(loginResponse)
	}
}

func (state *RuntimeState) userBootstrapOtp(profile *userProfile,
	fromCache bool) string {
	if len(profile.U2fAuthData) > 0 || len(profile.TOTPAuthData) > 0 {
		return ""
	}
	if fromCache { // Since we will want to clear the OTP, require connection.
		return ""
	}
	if profile.BootstrapOTP.Value == "" {
		return ""
	}
	if time.Since(profile.BootstrapOTP.ExpiresAt) >= 0 {
		return ""
	}
	return profile.BootstrapOTP.Value
}
<commit_msg>Change an error status code.<commit_after>package main

import (
	"crypto/subtle"
	"encoding/json"
	"net/http"
	"time"

	"github.com/Cloud-Foundations/keymaster/lib/instrumentedwriter"
	"github.com/Cloud-Foundations/keymaster/lib/webapi/v0/proto"
)

const bootstrapOtpAuthPath = "/api/v0/bootstrapOtpAuth"

func (state *RuntimeState) BootstrapOtpAuthHandler(w http.ResponseWriter,
	r *http.Request) {
	if state.sendFailureToClientIfLocked(w, r) {
		return
	}
	if r.Method != "POST" {
		state.writeFailureResponse(w, r, http.StatusMethodNotAllowed, "")
		return
	}
	state.logger.Debugf(3, "Got client POST connection")
	if err := r.ParseForm(); err != nil {
		state.logger.Println(err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError,
			"Error parsing form")
		return
	}
	authUser, currentAuthLevel, err := state.checkAuth(w, r, AuthTypeAny)
	if err != nil {
		state.logger.Debugf(1, "%v", err)
		return
	}
	w.(*instrumentedwriter.LoggingWriter).SetUsername(authUser)
	var inputOTP string
	if val, ok := r.Form["OTP"]; ok {
		if len(val) > 1 {
			state.writeFailureResponse(w, r, http.StatusBadRequest,
				"Just one OTP Value allowed")
			state.logger.Printf("Login with multiple OTP Values")
			return
		}
		inputOTP = val[0]
	}
	profile, _, fromCache, err := state.LoadUserProfile(authUser)
	if err != nil {
		state.logger.Printf("error loading user profile err=%s", err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError,
			"Failure loading user profile")
		return
	}
	if fromCache {
		state.writeFailureResponse(w, r, http.StatusServiceUnavailable,
			"Working in DB disconnected mode, try again later")
		return
	}
	requiredOTP := state.userBootstrapOtp(profile, fromCache)
	if requiredOTP == "" {
		state.writeFailureResponse(w, r, http.StatusPreconditionFailed,
			"No valid Bootstrap OTP saved")
		return
	}
	if subtle.ConstantTimeCompare([]byte(inputOTP), []byte(requiredOTP)) != 1 {
		state.logger.Debugf(0, "Invalid Bootstrap OTP value for %s\n",
			authUser)
		state.logger.Debugf(4, "  input: \"%s\" required: \"%s\"\n",
			inputOTP, requiredOTP)
		state.writeFailureResponse(w, r, http.StatusUnauthorized,
			"Invalid Bootstrap OTP")
		return
	}
	profile.BootstrapOTP = bootstrapOTPData{}
	if err := state.SaveUserProfile(authUser, profile); err != nil {
		state.logger.Printf("error saving profile randr=%s", err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError, "")
		return
	}
	_, err = state.updateAuthCookieAuthlevel(w, r,
		currentAuthLevel|AuthTypeBootstrapOTP)
	if err != nil {
		logger.Printf("Auth Cookie NOT found ? %s", err)
		state.writeFailureResponse(w, r, http.StatusInternalServerError,
			"Failure when validating Boostrap OTP")
		return
	}
	// eventNotifier.PublishBootstrapOtpAuthEvent(eventmon.AuthTypeBootstrapOTP,
	// authUser)
	// Now we send the user to the appropriate place
	returnAcceptType := getPreferredAcceptType(r)
	// TODO: The cert backend should depend also on per user preferences.
	loginResponse := proto.LoginResponse{Message: "success"}
	switch returnAcceptType {
	case "text/html":
		loginDestination := getLoginDestination(r)
		eventNotifier.PublishWebLoginEvent(authUser)
		state.logger.Debugf(0, "redirecting to: %s\n", loginDestination)
		http.Redirect(w, r, loginDestination, 302)
	default:
		w.WriteHeader(http.StatusOK)
		json.NewEncoder(w).Encode(loginResponse)
	}
}

func (state *RuntimeState) userBootstrapOtp(profile *userProfile,
	fromCache bool) string {
	if len(profile.U2fAuthData) > 0 || len(profile.TOTPAuthData) > 0 {
		return ""
	}
	if fromCache { // Since we will want to clear the OTP, require connection.
		return ""
	}
	if profile.BootstrapOTP.Value == "" {
		return ""
	}
	if time.Since(profile.BootstrapOTP.ExpiresAt) >= 0 {
		return ""
	}
	return profile.BootstrapOTP.Value
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	log "github.com/Sirupsen/logrus"
	flag "github.com/docker/docker/pkg/mflag"
	logx "github.com/mistifyio/mistify-logrus-ext"
	"github.com/mistifyio/mistify-operator-admin"
	"github.com/mistifyio/mistify-operator-admin/config"
	"github.com/mistifyio/mistify-operator-admin/db"
	"github.com/mistifyio/mistify-operator-admin/metrics"
)

func main() {
	var port uint
	var configFile, logLevel, statsd string
	var h bool

	flag.BoolVar(&h, []string{"h", "#help", "-help"}, false, "display the help")
	flag.UintVar(&port, []string{"p", "#port", "-port"}, 15000, "listen port")
	flag.StringVar(&configFile, []string{"c", "#config-file", "-config-file"}, "", "config file")
	flag.StringVar(&logLevel, []string{"l", "#log-level", "-log-level"}, "warning", "log level: debug/info/warning/error/critical/fatal")
	flag.StringVar(&statsd, []string{"s", "#statsd", "-statsd"}, "", "statsd address")
	flag.Parse()

	if h {
		flag.PrintDefaults()
		os.Exit(0)
	}

	err := logx.DefaultSetup(logLevel)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"func":  "log.ParseLevel",
		}).Fatal("Could not parse log level")
	}

	if configFile == "" {
		log.Fatal("need a config file")
	}

	if err := config.Load(configFile); err != nil {
		log.Fatal(err)
	}

	if statsd != "" {
		conf := config.Get()
		conf.Metrics.StatsdAddress = statsd
	}

	if err = metrics.LoadContext(); err != nil {
		log.Warning(err)
	}

	_, err = db.Connect(nil)
	if err != nil {
		log.Fatal(err)
	}
	operator.Run(port)
}
<commit_msg>[MIST-372] Idiomatic error check<commit_after>package main

import (
	"os"

	log "github.com/Sirupsen/logrus"
	flag "github.com/docker/docker/pkg/mflag"
	logx "github.com/mistifyio/mistify-logrus-ext"
	"github.com/mistifyio/mistify-operator-admin"
	"github.com/mistifyio/mistify-operator-admin/config"
	"github.com/mistifyio/mistify-operator-admin/db"
	"github.com/mistifyio/mistify-operator-admin/metrics"
)

func main() {
	var port uint
	var configFile, logLevel, statsd string
	var h bool

	flag.BoolVar(&h, []string{"h", "#help", "-help"}, false, "display the help")
	flag.UintVar(&port, []string{"p", "#port", "-port"}, 15000, "listen port")
	flag.StringVar(&configFile, []string{"c", "#config-file", "-config-file"}, "", "config file")
	flag.StringVar(&logLevel, []string{"l", "#log-level", "-log-level"}, "warning", "log level: debug/info/warning/error/critical/fatal")
	flag.StringVar(&statsd, []string{"s", "#statsd", "-statsd"}, "", "statsd address")
	flag.Parse()

	if h {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if err := logx.DefaultSetup(logLevel); err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"func":  "log.ParseLevel",
		}).Fatal("Could not parse log level")
	}

	if configFile == "" {
		log.Fatal("need a config file")
	}

	if err := config.Load(configFile); err != nil {
		log.Fatal(err)
	}

	if statsd != "" {
		conf := config.Get()
		conf.Metrics.StatsdAddress = statsd
	}

	if err = metrics.LoadContext(); err != nil {
		log.Warning(err)
	}

	_, err = db.Connect(nil)
	if err != nil {
		log.Fatal(err)
	}
	operator.Run(port)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	ss "github.com/shadowsocks/shadowsocks-go/shadowsocks"
	"io"
	"math"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var config struct {
	server string
	port   int
	passwd string
	core   int
	nconn  int
	nreq   int
	// nsec   int
}

var debug ss.DebugLog

func doOneRequest(client *http.Client, uri string, buf []byte) (err error) {
	resp, err := client.Get(uri)
	if err != nil {
		fmt.Printf("GET %s error: %v\n", uri, err)
		return err
	}
	for err == nil {
		_, err = resp.Body.Read(buf)
		if debug {
			debug.Println(string(buf))
		}
	}
	if err != io.EOF {
		fmt.Printf("Read %s response error: %v\n", uri, err)
	} else {
		err = nil
	}
	return
}

func get(connid int, uri, serverAddr string, rawAddr []byte, enctbl *ss.EncryptTable, done chan []time.Duration) {
	reqDone := 0
	reqTime := make([]time.Duration, config.nreq, config.nreq)
	defer func() {
		done <- reqTime[:reqDone]
	}()
	tr := &http.Transport{
		Dial: func(_, _ string) (net.Conn, error) {
			return ss.DialWithRawAddr(rawAddr, serverAddr, enctbl)
		},
	}

	buf := make([]byte, 8192)
	client := &http.Client{Transport: tr}
	for ; reqDone < config.nreq; reqDone++ {
		start := time.Now()
		if err := doOneRequest(client, uri, buf); err != nil {
			return
		}
		reqTime[reqDone] = time.Now().Sub(start)

		if (reqDone+1)%1000 == 0 {
			fmt.Printf("conn %d finished %d get requests\n", connid, reqDone+1)
		}
	}
}

func main() {
	flag.StringVar(&config.server, "s", "127.0.0.1", "server:port")
	flag.IntVar(&config.port, "p", 0, "server:port")
	flag.IntVar(&config.core, "core", 1, "number of CPU cores to use")
	flag.StringVar(&config.passwd, "k", "", "password")
	flag.IntVar(&config.nconn, "nc", 1, "number of connection to server")
	flag.IntVar(&config.nreq, "nr", 1, "number of request for each connection")
	// flag.IntVar(&config.nsec, "ns", 0, "run how many seconds for each connection")
	flag.BoolVar((*bool)(&debug), "d", false, "print http response body for debugging")

	flag.Parse()

	if config.server == "" || config.port == 0 || config.passwd == "" || len(flag.Args()) != 1 {
		fmt.Printf("Usage: %s -s <server> -p <port> -k <password> <url>\n", os.Args[0])
		os.Exit(1)
	}

	runtime.GOMAXPROCS(config.core)
	uri := flag.Arg(0)
	if !strings.HasPrefix(uri, "http://") {
		uri = "http://" + uri
	}

	enctbl := ss.GetTable(config.passwd)
	serverAddr := net.JoinHostPort(config.server, strconv.Itoa(config.port))

	parsedURL, err := url.Parse(uri)
	if err != nil {
		fmt.Println("Error parsing url:", err)
		os.Exit(1)
	}
	host, port, _ := net.SplitHostPort(parsedURL.Host)
	if port == "" {
		host += ":80"
	} else {
		host = parsedURL.Host
	}
	fmt.Println(host)
	rawAddr, err := ss.RawAddr(host)
	if err != nil {
		panic("Error getting raw address.")
		return
	}

	done := make(chan []time.Duration)
	for i := 1; i <= config.nconn; i++ {
		go get(i, uri, serverAddr, rawAddr, enctbl, done)
	}

	// collect request finish time
	reqTime := make([]int64, config.nconn*config.nreq)
	reqDone := 0
	for i := 1; i <= config.nconn; i++ {
		rt := <-done
		for _, t := range rt {
			reqTime[reqDone] = int64(t)
			reqDone++
		}
	}

	fmt.Println("number of total requests:", config.nconn*config.nreq)
	fmt.Println("number of finished requests:", reqDone)
	if reqDone == 0 {
		return
	}

	// calculate average an standard deviation
	reqTime = reqTime[:reqDone]
	var sum int64
	for _, d := range reqTime {
		sum += d
	}
	avg := float64(sum) / float64(reqDone)

	varSum := float64(0)
	for _, d := range reqTime {
		di := math.Abs(float64(d) - avg)
		di *= di
		varSum += di
	}
	stddev := math.Sqrt(varSum / float64(reqDone))
	fmt.Println("\ntotal time used:", time.Duration(sum))
	fmt.Println("average time per request:", time.Duration(avg))
	fmt.Println("standard deviation:", time.Duration(stddev))
}
<commit_msg>Use net.JoinHostPort to joint host and port in benchmark.<commit_after>package main

import (
	"flag"
	"fmt"
	ss "github.com/shadowsocks/shadowsocks-go/shadowsocks"
	"io"
	"math"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var config struct {
	server string
	port   int
	passwd string
	core   int
	nconn  int
	nreq   int
	// nsec   int
}

var debug ss.DebugLog

func doOneRequest(client *http.Client, uri string, buf []byte) (err error) {
	resp, err := client.Get(uri)
	if err != nil {
		fmt.Printf("GET %s error: %v\n", uri, err)
		return err
	}
	for err == nil {
		_, err = resp.Body.Read(buf)
		if debug {
			debug.Println(string(buf))
		}
	}
	if err != io.EOF {
		fmt.Printf("Read %s response error: %v\n", uri, err)
	} else {
		err = nil
	}
	return
}

func get(connid int, uri, serverAddr string, rawAddr []byte, enctbl *ss.EncryptTable, done chan []time.Duration) {
	reqDone := 0
	reqTime := make([]time.Duration, config.nreq, config.nreq)
	defer func() {
		done <- reqTime[:reqDone]
	}()
	tr := &http.Transport{
		Dial: func(_, _ string) (net.Conn, error) {
			return ss.DialWithRawAddr(rawAddr, serverAddr, enctbl)
		},
	}

	buf := make([]byte, 8192)
	client := &http.Client{Transport: tr}
	for ; reqDone < config.nreq; reqDone++ {
		start := time.Now()
		if err := doOneRequest(client, uri, buf); err != nil {
			return
		}
		reqTime[reqDone] = time.Now().Sub(start)

		if (reqDone+1)%1000 == 0 {
			fmt.Printf("conn %d finished %d get requests\n", connid, reqDone+1)
		}
	}
}

func main() {
	flag.StringVar(&config.server, "s", "127.0.0.1", "server:port")
	flag.IntVar(&config.port, "p", 0, "server:port")
	flag.IntVar(&config.core, "core", 1, "number of CPU cores to use")
	flag.StringVar(&config.passwd, "k", "", "password")
	flag.IntVar(&config.nconn, "nc", 1, "number of connection to server")
	flag.IntVar(&config.nreq, "nr", 1, "number of request for each connection")
	// flag.IntVar(&config.nsec, "ns", 0, "run how many seconds for each connection")
	flag.BoolVar((*bool)(&debug), "d", false, "print http response body for debugging")

	flag.Parse()

	if config.server == "" || config.port == 0 || config.passwd == "" || len(flag.Args()) != 1 {
		fmt.Printf("Usage: %s -s <server> -p <port> -k <password> <url>\n", os.Args[0])
		os.Exit(1)
	}

	runtime.GOMAXPROCS(config.core)
	uri := flag.Arg(0)
	if !strings.HasPrefix(uri, "http://") {
		uri = "http://" + uri
	}

	enctbl := ss.GetTable(config.passwd)
	serverAddr := net.JoinHostPort(config.server, strconv.Itoa(config.port))

	parsedURL, err := url.Parse(uri)
	if err != nil {
		fmt.Println("Error parsing url:", err)
		os.Exit(1)
	}
	host, _, err := net.SplitHostPort(parsedURL.Host)
	if err != nil {
		host = net.JoinHostPort(parsedURL.Host, "80")
	} else {
		host = parsedURL.Host
	}
	// fmt.Println(host)
	rawAddr, err := ss.RawAddr(host)
	if err != nil {
		panic("Error getting raw address.")
		return
	}

	done := make(chan []time.Duration)
	for i := 1; i <= config.nconn; i++ {
		go get(i, uri, serverAddr, rawAddr, enctbl, done)
	}

	// collect request finish time
	reqTime := make([]int64, config.nconn*config.nreq)
	reqDone := 0
	for i := 1; i <= config.nconn; i++ {
		rt := <-done
		for _, t := range rt {
			reqTime[reqDone] = int64(t)
			reqDone++
		}
	}

	fmt.Println("number of total requests:", config.nconn*config.nreq)
	fmt.Println("number of finished requests:", reqDone)
	if reqDone == 0 {
		return
	}

	// calculate average an standard deviation
	reqTime = reqTime[:reqDone]
	var sum int64
	for _, d := range reqTime {
		sum += d
	}
	avg := float64(sum) / float64(reqDone)

	varSum := float64(0)
	for _, d := range reqTime {
		di := math.Abs(float64(d) - avg)
		di *= di
		varSum += di
	}
	stddev := math.Sqrt(varSum / float64(reqDone))
	fmt.Println("\ntotal time used:", time.Duration(sum))
	fmt.Println("average time per request:", time.Duration(avg))
	fmt.Println("standard deviation:", time.Duration(stddev))
}
<|endoftext|>