<commit_before>//  Copyright 2016 The goscope Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package main

import (
	"image"
	"image/color"
	"image/png"
	"log"
	"os"
	"sort"

	"github.com/zagrodzki/goscope/dummy"
	"github.com/zagrodzki/goscope/scope"
)

type aggrPoint struct {
	sumY  int
	sizeY int
}

func (p aggrPoint) add(y int) aggrPoint {
	p.sumY += y
	p.sizeY++
	return p
}

func (p aggrPoint) toPoint(x int) image.Point {
	return image.Point{x, p.sumY / p.sizeY}
}

type floatRange struct {
	min float64
	max float64
}

func (r floatRange) width() float64 {
	return r.max - r.min
}

func samplesToPoints(samples []scope.Sample, sampleRangeY floatRange, start, end image.Point) []image.Point {
	if len(samples) == 0 {
		return nil
	}

	sampleWidthX := float64(len(samples) - 1)
	sampleWidthY := sampleRangeY.width()

	pixelStartX := float64(start.X)
	pixelEndY := float64(end.Y)
	pixelWidthX := float64(end.X - start.X)
	pixelWidthY := float64(end.Y - start.Y)

	aggrPoints := make(map[int]aggrPoint)
	for i, y := range samples {
		mapX := int(pixelStartX + float64(i)/sampleWidthX*pixelWidthX)
		mapY := int(pixelEndY - float64(y-scope.Sample(sampleRangeY.min))/sampleWidthY*pixelWidthY)
		aggrPoints[mapX] = aggrPoints[mapX].add(mapY)
	}
	var points []image.Point
	for x, p := range aggrPoints {
		points = append(points, p.toPoint(x))
	}

	return points
}

// Plot represents the entire plotting area.
type Plot struct {
	*image.RGBA
}

// Fill fills the plot with a color.
func (plot Plot) Fill(col color.RGBA) {
	bounds := plot.Bounds()
	for i := bounds.Min.X; i < bounds.Max.X; i++ {
		for j := bounds.Min.Y; j < bounds.Max.Y; j++ {
			plot.Set(i, j, col)
		}
	}
}

// DrawLine draws a straight line from pixel p1 to p2.
func (plot Plot) DrawLine(p1, p2 image.Point, col color.RGBA) {
	if p1.X == p2.X { // vertical line
		for i := min(p1.Y, p2.Y); i <= max(p1.Y, p2.Y); i++ {
			plot.Set(p1.X, i, col)
		}
		return
	}

	// Calculating the parameters of the equation
	// of the straight line (in the form y=a*x+b)
	// passing through p1 and p2.

	// slope of the line
	a := float64(p1.Y-p2.Y) / float64(p1.X-p2.X)
	// intercept of the line
	b := float64(p1.Y) - float64(p1.X)*a

	// To avoid visual "gaps" between the pixels we switch on,
	// we draw the line in one of two ways.
	if abs(p1.X-p2.X) >= abs(p1.Y-p2.Y) {
		// If the line is more horizontal than vertical,
		// for every pixel column between p1 and p2
		// we find and switch on the pixel closest to y=a*x+b
		for i := min(p1.X, p2.X); i <= max(p1.X, p2.X); i++ {
			plot.Set(i, int(a*float64(i)+b), col)
		}
	} else {
		// If the line is more vertical than horizontal,
		// for every pixel row between p1 and p2
		// we find and switch on the pixel closest to y=a*x+b
		for i := min(p1.Y, p2.Y); i <= max(p1.Y, p2.Y); i++ {
			plot.Set(int((float64(i)-b)/a), i, col)
		}
	}
}

// DrawSamples draws samples in the image rectangle defined by
// starting (upper left) and ending (lower right) pixel.
func (plot Plot) DrawSamples(samples []scope.Sample, sampleRangeY floatRange, start, end image.Point, col color.RGBA) {
	points := samplesToPoints(samples, sampleRangeY, start, end)
	sort.Sort(pointsByX(points))
	for i := 1; i < len(points); i++ {
		plot.DrawLine(points[i-1], points[i], col)
	}
}

// DrawAll draws samples from all the channels into one image.
func (plot Plot) DrawAll(samples map[scope.ChanID][]scope.Sample, ranges map[scope.ChanID]floatRange, cols map[scope.ChanID]color.RGBA) {
	b := plot.Bounds()
	x1 := b.Min.X + 10
	x2 := b.Max.X - 10
	y1 := b.Min.Y + 10
	y2 := b.Min.Y + 10 + int((b.Max.Y-b.Min.Y-10*(len(samples)+1))/len(samples))
	step := y2 - b.Min.Y
	for id, v := range samples {
		plot.DrawSamples(v, ranges[id], image.Point{x1, y1}, image.Point{x2, y2}, cols[id])
		y1 = y1 + step
		y2 = y2 + step
	}
}

func plot(dev scope.Device, outputFile string) error {
	data, stop, err := dev.StartSampling()
	defer stop()
	if err != nil {
		return err
	}
	samples := (<-data).Samples

	plot := Plot{image.NewRGBA(image.Rect(0, 0, 800, 600))}
	colWhite := color.RGBA{255, 255, 255, 255}
	colRed := color.RGBA{255, 0, 0, 255}
	colGreen := color.RGBA{0, 255, 0, 255}
	colBlue := color.RGBA{0, 0, 255, 255}
	colBlack := color.RGBA{0, 0, 0, 255}
	chanCols := [4]color.RGBA{colRed, colGreen, colBlue, colBlack}

	plot.Fill(colWhite)

	ranges := make(map[scope.ChanID]floatRange)
	cols := make(map[scope.ChanID]color.RGBA)
	next := 0
	for _, id := range dev.Channels() {
		ranges[id] = floatRange{-1, 1}
		cols[id] = chanCols[next]
		next = (next + 1) % 4
	}
	plot.DrawAll(samples, ranges, cols)

	f, err := os.Create(outputFile)
	if err != nil {
		return err
	}
	defer f.Close()
	png.Encode(f, plot)
	return nil
}

type pointsByX []image.Point

func (a pointsByX) Len() int {
	return len(a)
}

func (a pointsByX) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a pointsByX) Less(i, j int) bool {
	return a[i].X < a[j].X
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func abs(a int) int {
	if a < 0 {
		return -a
	}
	return a
}

func main() {
	dev, err := dummy.Open("")
	if err != nil {
		log.Fatalf("Cannot open the device: %v", err)
	}
	if err := plot(dev, "draw.png"); err != nil {
		log.Fatalf("Cannot plot samples: %v", err)
	}
}
<commit_msg>Y ranges passed through cmd flags<commit_after>//  Copyright 2016 The goscope Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/zagrodzki/goscope/dummy"
	"github.com/zagrodzki/goscope/scope"
)

type aggrPoint struct {
	sumY  int
	sizeY int
}

func (p aggrPoint) add(y int) aggrPoint {
	p.sumY += y
	p.sizeY++
	return p
}

func (p aggrPoint) toPoint(x int) image.Point {
	return image.Point{x, p.sumY / p.sizeY}
}

type floatRange struct {
	min float64
	max float64
}

func (r floatRange) width() float64 {
	return r.max - r.min
}

func samplesToPoints(samples []scope.Sample, sampleRangeY floatRange, start, end image.Point) []image.Point {
	if len(samples) == 0 {
		return nil
	}

	sampleWidthX := float64(len(samples) - 1)
	sampleWidthY := sampleRangeY.width()

	pixelStartX := float64(start.X)
	pixelEndY := float64(end.Y)
	pixelWidthX := float64(end.X - start.X)
	pixelWidthY := float64(end.Y - start.Y)

	aggrPoints := make(map[int]aggrPoint)
	for i, y := range samples {
		mapX := int(pixelStartX + float64(i)/sampleWidthX*pixelWidthX)
		mapY := int(pixelEndY - float64(y-scope.Sample(sampleRangeY.min))/sampleWidthY*pixelWidthY)
		aggrPoints[mapX] = aggrPoints[mapX].add(mapY)
	}
	var points []image.Point
	for x, p := range aggrPoints {
		points = append(points, p.toPoint(x))
	}

	return points
}

// Plot represents the entire plotting area.
type Plot struct {
	*image.RGBA
}

// Fill fills the plot with a color.
func (plot Plot) Fill(col color.RGBA) {
	bounds := plot.Bounds()
	for i := bounds.Min.X; i < bounds.Max.X; i++ {
		for j := bounds.Min.Y; j < bounds.Max.Y; j++ {
			plot.Set(i, j, col)
		}
	}
}

// DrawLine draws a straight line from pixel p1 to p2.
func (plot Plot) DrawLine(p1, p2 image.Point, col color.RGBA) {
	if p1.X == p2.X { // vertical line
		for i := min(p1.Y, p2.Y); i <= max(p1.Y, p2.Y); i++ {
			plot.Set(p1.X, i, col)
		}
		return
	}

	// Calculating the parameters of the equation
	// of the straight line (in the form y=a*x+b)
	// passing through p1 and p2.

	// slope of the line
	a := float64(p1.Y-p2.Y) / float64(p1.X-p2.X)
	// intercept of the line
	b := float64(p1.Y) - float64(p1.X)*a

	// To avoid visual "gaps" between the pixels we switch on,
	// we draw the line in one of two ways.
	if abs(p1.X-p2.X) >= abs(p1.Y-p2.Y) {
		// If the line is more horizontal than vertical,
		// for every pixel column between p1 and p2
		// we find and switch on the pixel closest to y=a*x+b
		for i := min(p1.X, p2.X); i <= max(p1.X, p2.X); i++ {
			plot.Set(i, int(a*float64(i)+b), col)
		}
	} else {
		// If the line is more vertical than horizontal,
		// for every pixel row between p1 and p2
		// we find and switch on the pixel closest to y=a*x+b
		for i := min(p1.Y, p2.Y); i <= max(p1.Y, p2.Y); i++ {
			plot.Set(int((float64(i)-b)/a), i, col)
		}
	}
}

// DrawSamples draws samples in the image rectangle defined by
// starting (upper left) and ending (lower right) pixel.
func (plot Plot) DrawSamples(samples []scope.Sample, sampleRangeY floatRange, start, end image.Point, col color.RGBA) {
	points := samplesToPoints(samples, sampleRangeY, start, end)
	sort.Sort(pointsByX(points))
	for i := 1; i < len(points); i++ {
		plot.DrawLine(points[i-1], points[i], col)
	}
}

// DrawAll draws samples from all the channels into one image.
func (plot Plot) DrawAll(samples map[scope.ChanID][]scope.Sample, ranges map[scope.ChanID]floatRange, cols map[scope.ChanID]color.RGBA) {
	b := plot.Bounds()
	x1 := b.Min.X + 10
	x2 := b.Max.X - 10
	y1 := b.Min.Y + 10
	y2 := b.Min.Y + 10 + int((b.Max.Y-b.Min.Y-10*(len(samples)+1))/len(samples))
	step := y2 - b.Min.Y
	for id, v := range samples {
		plot.DrawSamples(v, ranges[id], image.Point{x1, y1}, image.Point{x2, y2}, cols[id])
		y1 = y1 + step
		y2 = y2 + step
	}
}

func plot(dev scope.Device, ranges map[scope.ChanID]floatRange, outputFile string) error {
	data, stop, err := dev.StartSampling()
	defer stop()
	if err != nil {
		return err
	}
	samples := (<-data).Samples

	plot := Plot{image.NewRGBA(image.Rect(0, 0, 800, 600))}
	colWhite := color.RGBA{255, 255, 255, 255}
	colRed := color.RGBA{255, 0, 0, 255}
	colGreen := color.RGBA{0, 255, 0, 255}
	colBlue := color.RGBA{0, 0, 255, 255}
	colBlack := color.RGBA{0, 0, 0, 255}
	chanCols := [4]color.RGBA{colRed, colGreen, colBlue, colBlack}

	plot.Fill(colWhite)

	cols := make(map[scope.ChanID]color.RGBA)
	next := 0
	for _, id := range dev.Channels() {
		if _, exists := ranges[id]; !exists {
			ranges[id] = floatRange{-1, 1}
		}
		cols[id] = chanCols[next]
		next = (next + 1) % 4
	}
	plot.DrawAll(samples, ranges, cols)

	f, err := os.Create(outputFile)
	if err != nil {
		return err
	}
	defer f.Close()
	png.Encode(f, plot)
	return nil
}

type pointsByX []image.Point

func (a pointsByX) Len() int {
	return len(a)
}

func (a pointsByX) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a pointsByX) Less(i, j int) bool {
	return a[i].X < a[j].X
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func abs(a int) int {
	if a < 0 {
		return -a
	}
	return a
}

type chanRanges map[scope.ChanID]floatRange

func (i *chanRanges) String() string {
	return fmt.Sprintf("%v", *i)
}

func (i *chanRanges) Set(value string) error {
	parts := strings.Split(value, ":")
	if len(parts) != 2 {
		return errors.New("use format: \"chanID:lowerY,upperY\"")
	}
	numbers := strings.Split(parts[1], ",")
	if len(numbers) != 2 {
		return errors.New("use format: \"chanID:lowerY,upperY\"")
	}
	min, err := strconv.ParseFloat(numbers[0], 64)
	if err != nil {
		return err
	}
	max, err := strconv.ParseFloat(numbers[1], 64)
	if err != nil {
		return err
	}
	(*i)[scope.ChanID(parts[0])] = floatRange{min, max}
	return nil
}

func main() {
	dev, err := dummy.Open("")
	if err != nil {
		log.Fatalf("Cannot open the device: %v", err)
	}

	fileName := flag.String("file", "draw.png", "output file name")
	ranges := chanRanges{}
	flag.Var(&ranges, "range", "channel Y range, format: \"chanID:lowerY,upperY\"")
	flag.Parse()

	if err := plot(dev, ranges, *fileName); err != nil {
		log.Fatalf("Cannot plot samples: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2010-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"github.com/hoisie/web"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"strings"
)

var contextDir = ""

func globalServe(params *web.Context, val string) string {
	cssPath := params.Params["style"]
	if len(cssPath) == 0 {
		cssPath = "default.css"
	} else {
		cssPath += ".css"
	}
	file, err := ioutil.ReadFile("/" + val)
	if err != nil {
		return "404: File not found: doc-g/" + val
	}
	if strings.HasSuffix(val, ".md") {
		return string(blackfriday.MarkdownBasic(file))
	}
	return string(file)
}

func contextServe(params *web.Context, val string) string {
	cssPath := params.Params["style"]
	if len(cssPath) == 0 {
		cssPath = "default.css"
	} else {
		cssPath += ".css"
	}
	if len(val) == 0 || (len(val) == 1 && val == "/") {
		val = "index.md"
	}
	file, err := ioutil.ReadFile(contextDir + val)
	if err != nil {
		return "404: File not found: doc/" + val
	}
	if strings.HasSuffix(val, ".md") {
		return string(blackfriday.MarkdownBasic(file))
	}
	return string(file)
}

func main() {
	global := flag.Bool("global", false, "Allow the server to access any files that the user running it has access to.")
	remote := flag.Bool("remote", false, "Allow the remote clients to access the server.")
	port := "15448"
	settingsFile, err := ioutil.ReadFile("docserver.conf")
	if err == nil {
		settings := strings.Split(string(settingsFile), "\n")
		for i := 0; i < len(settings); i++ {
			if strings.HasPrefix(settings[i], "Port:") {
				port = strings.Trim(strings.Replace(settings[i], "Port:", "", 1), "\t ")
			} else if strings.HasPrefix(settings[i], "Context:") {
				contextDir = strings.Trim(strings.Replace(settings[i], "Context:", "", 1), "\t ")
			} else if strings.HasPrefix(settings[i], "Global:") {
				g := strings.Trim(strings.Replace(settings[i], "Global:", "", 1), "\t ")
				if strings.ToLower(g) == "true" {
					*global = true
				}
			} else if strings.HasPrefix(settings[i], "AllowRemote:") {
				g := strings.Trim(strings.Replace(settings[i], "AllowRemote:", "", 1), "\t ")
				if strings.ToLower(g) == "true" {
					*remote = true
				}
			}
		}
	}
	flag.Parse()
	//read the context from the input and override whats in the settings file if something was there
	if flag.NArg() != 0 {
		contextDir = flag.Arg(0)
	}
	//make sure the context is a directory
	if len(contextDir) != 0 && !strings.HasSuffix(contextDir, "/") {
		contextDir += "/"
	}
	web.Get("/doc/(.*)", contextServe)
	web.Get("/doc", contextServe)
	if *global {
		web.Get("/doc-g/(.*)", globalServe)
		web.Get("/doc-g", globalServe)
	}
	if *remote {
		web.Run("0.0.0.0:" + port)
	} else {
		web.Run("127.0.0.1:" + port)
	}
}
<commit_msg>Cleaned up code and added title element.<commit_after>/*
   Copyright 2010-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"github.com/hoisie/web"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"strings"
)

func mkServer(contextDir string) func(*web.Context, string) string {
	return func(params *web.Context, val string) string {
		if len(val) == 0 || (len(val) == 1 && val == "/") {
			val = "index.md"
		}
		file, err := ioutil.ReadFile(contextDir + val)
		if err != nil {
			return "404: File not found: " + val
		}
		if strings.HasSuffix(val, ".md") {
			return `<html>
	<head>
		<title>` + val + `</title>
	</head>
	<body>
` + string(blackfriday.MarkdownBasic(file)) + `
	</body>
</html>`
		}
		return string(file)
	}
}

func main() {
	var contextDir = ""
	global := flag.Bool("global", false, "Allow the server to access any files that the user running it has access to.")
	remote := flag.Bool("remote", false, "Allow the remote clients to access the server.")
	port := "15448"
	flag.Parse()
	//read the context from the input and override whats in the settings file if something was there
	if flag.NArg() != 0 {
		contextDir = flag.Arg(0)
	}
	//make sure the context is a directory
	if len(contextDir) != 0 && !strings.HasSuffix(contextDir, "/") {
		contextDir += "/"
	}

	contextServe := mkServer(contextDir)
	globalServe := mkServer("")
	web.Get("/doc/(.*)", contextServe)
	web.Get("/doc", contextServe)
	if *global {
		web.Get("/doc-g/(.*)", globalServe)
		web.Get("/doc-g", globalServe)
	}
	if *remote {
		web.Run("0.0.0.0:" + port)
	} else {
		web.Run("127.0.0.1:" + port)
	}
}
<|endoftext|>


<commit_before>package narcissus

import (
	"testing"

	"honnef.co/go/augeas"
)

func TestPasswd(t *testing.T) {
	aug, err := augeas.New(fakeroot, "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)

	passwd, err := n.NewPasswd()
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	if len(passwd.Users) != 42 {
		t.Errorf("Expected 42 users, got %v", len(passwd.Users))
	}

	if passwd.Users["raphink"].UID != 1000 {
		t.Errorf("Expected user raphink to have uid 1000, got %v", passwd.Users["raphink"].UID)
	}
}

func TestPasswdUser(t *testing.T) {
	aug, err := augeas.New(fakeroot, "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)

	// Test one fstab
	user, err := n.NewPasswdUser("raphink")
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	if user.Account != "raphink" {
		t.Errorf("Expected account to be raphink, got %s", user.Account)
	}

	if user.UID != 1000 {
		t.Errorf("Expected uid to be 1000, got %v", user.UID)
	}

}
<commit_msg>Writing tests for passwd<commit_after>package narcissus

import (
	"testing"

	"honnef.co/go/augeas"
)

func TestParsePasswd(t *testing.T) {
	aug, err := augeas.New(fakeroot, "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)

	passwd, err := n.NewPasswd()
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	if len(passwd.Users) != 42 {
		t.Errorf("Expected 42 users, got %v", len(passwd.Users))
	}

	if passwd.Users["raphink"].UID != 1000 {
		t.Errorf("Expected user raphink to have uid 1000, got %v", passwd.Users["raphink"].UID)
	}
}

func TestParsePasswdUser(t *testing.T) {
	aug, err := augeas.New(fakeroot, "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)

	// Test one fstab
	user, err := n.NewPasswdUser("raphink")
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	if user.Account != "raphink" {
		t.Errorf("Expected account to be raphink, got %s", user.Account)
	}

	if user.UID != 1000 {
		t.Errorf("Expected uid to be 1000, got %v", user.UID)
	}

}

func TestWritePasswd(t *testing.T) {
	aug, err := augeas.New(fakeroot, "", augeas.None)
	if err != nil {
		t.Fatal("Failed to create Augeas handler")
	}
	n := New(&aug)

	passwd, err := n.NewPasswd()
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	err = n.Write(passwd)
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}
	err = aug.Save()
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	// TODO: check that file is unchanged

	var user = passwd.Users["raphink"]
	user.Shell = "/bin/sh"
	passwd.Users["raphink"] = user

	err = n.Write(passwd)
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}
	err = aug.Save()
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}

	t.Skip("Saving fails for now")

	errStr, _ := aug.Get("/augeas//error/message")
	if errStr != "" {
		t.Errorf("Failed with %s", errStr)
	}
}
<|endoftext|>


<commit_before>package main

type sourceConfig struct {
	name           string            "json:name"
	upstreamDriver string            "json:type"
	upstream       upstreamInterface "json:upstream"
	version        versionType       "json:version"
	output         []outputType      "json:specFile"
}

type upstreamTarget interface {
	func check()bool
	func download() error
	func build() error
}

type versionType struct {
	fmt  string   "json:fmt"
	cur  string   "json:cur"
	past []string "json:past"
}

type outputType struct {
	active      bool   "json:active"
	gitRepo     string "json:gitRepo"
	location    string "json:location"
	outputID    int    "json:outputID"
	outputPrune int    "json:outputPrune"
}

type httpResponseTracker struct {
	driver string "json:driver"
	url    string "json:url"
}

func ServeSource() {
	var configs []sourceConfig

}
<commit_msg>dinked around with some stuff in the middle of being slammed - committing to get them in<commit_after>package main

import (
	"net/http"
	textTempl "text/template"
)

// type sourceConfig struct {
// 	name           string            "json:name"
// 	upstreamDriver string            "json:type"
// 	upstream       upstreamTarget "json:upstream"
// 	output         []outputType      "json:specFile"
// }

// type upstreamTarget interface {
// 	check() error
// 	version() string
// 	download() error
// }

type versionType struct {
	fmt  string   `json:"fmt"`
	cur  string   `json:"cur"`
	past []string `json:"past"`
}

func (v versionType) nextPossible() (possible []string ) {
	fmtChunks := strings.Split(v.fmt, "%v")
	remainingChunks := fmtChunks

	var possibleSubs []string
	for len(fmtChunks) > 0 {
		prefix := strings.TrimSuffix(v.cur, fmtChunks[len(fmtChunks)-1])

	}
}

type outputType struct {
	active      bool   `json:"active"`
	gitRepo     string `json:"gitRepo"`
	location    string `json:"location"`
	outputID    int    `json:"outputID"`
	outputPrune int    `json:"outputPrune"`
}

type httpResponseTracker struct {
	version versionType `json:"version"`
	driver string `json:"driver"`
	url    string `json:"url"`
}

// func (upstream httpResponseTracker) check() error {

// 	req, _ := http.NewRequest("GET", upstream.Replace(upstream.url), nil)

// }<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v2"
)

func Add(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	parent, err := getParent(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to get parent id: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	paths := strings.Split(strings.TrimPrefix(ps.ByName("filepath"), "/"), "/")

	file := &drive.File{
		Title:   paths[len(paths)-1],
		Parents: []*drive.ParentReference{parent},
	}

	if r.Header.Get("folder") == "true" {
		file.MimeType = "application/vnd.google-apps.folder"
	}

	_, err = srv.Files.Insert(file).Media(r.Body).Do()
	if err != nil {
		log.Printf("Error while uploading file: %v", err)
		http.Error(w, "Error while uploading file",
			http.StatusInternalServerError)
	}

	r.Body.Close()
}

func Browse(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	list, err := getDirectoryList(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to get list of directory: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	jsonResponse := make([]string, len(list))
	for i, v := range list {
		jsonResponse[i] = v.Title
	}

	j, err := json.Marshal(jsonResponse)
	if err != nil {
		log.Printf("Error while encoding JSON: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	fmt.Fprint(w, string(j))
}

func Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	file, err := getFileByPath(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusUnauthorized)
		return
	}

	err = srv.Files.Delete(file.Id).Do()
	if err != nil {
		log.Printf("Failed to get file: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
}

func NewUser(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	url := config.AuthCodeURL("todo", oauth2.AccessTypeOffline)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

func Publish(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	file, err := getFileByPath(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusNotFound)
		return
	}

	permission := &drive.Permission{
		Value: "",
		Type:  "anyone",
		Role:  "reader",
	}

	_, err = drive.NewPermissionsService(srv).Insert(file.Id, permission).Do()
	if err != nil {
		log.Printf("Unable to insert new permission: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	updated, err := srv.Files.Get(file.Id).Do()
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusNotFound)
		return
	}

	fmt.Fprint(w, updated.WebViewLink)
}

func Read(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	file, err := getFileByPath(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusUnauthorized)
		return
	}

	response, err := srv.Files.Get(file.Id).Download()
	if err != nil {
		log.Printf("Failed to get file: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	io.Copy(w, response.Body)
	response.Body.Close()
}

func Validate(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	code := r.FormValue("code")
	if code == "" {
		log.Println("No authorization code present")
		http.Error(w, "No authorization code present", http.StatusUnauthorized)
		return
	}

	token, err := config.Exchange(oauth2.NoContext, code)
	if err != nil {
		log.Printf("Unable to retrieve token: %v", err)
		http.Error(w, "Unable to retrieve token",
			http.StatusInternalServerError)
		return
	}

	fmt.Fprint(w, token.AccessToken)
}
<commit_msg>changed statuscode to the right one<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v2"
)

func Add(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	parent, err := getParent(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to get parent id: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	paths := strings.Split(strings.TrimPrefix(ps.ByName("filepath"), "/"), "/")

	file := &drive.File{
		Title:   paths[len(paths)-1],
		Parents: []*drive.ParentReference{parent},
	}

	if r.Header.Get("folder") == "true" {
		file.MimeType = "application/vnd.google-apps.folder"
	}

	_, err = srv.Files.Insert(file).Media(r.Body).Do()
	if err != nil {
		log.Printf("Error while uploading file: %v", err)
		http.Error(w, "Error while uploading file",
			http.StatusInternalServerError)
	}

	r.Body.Close()
}

func Browse(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	list, err := getDirectoryList(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to get list of directory: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	jsonResponse := make([]string, len(list))
	for i, v := range list {
		jsonResponse[i] = v.Title
	}

	j, err := json.Marshal(jsonResponse)
	if err != nil {
		log.Printf("Error while encoding JSON: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	fmt.Fprint(w, string(j))
}

func Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	file, err := getFileByPath(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusUnauthorized)
		return
	}

	err = srv.Files.Delete(file.Id).Do()
	if err != nil {
		log.Printf("Failed to get file: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
}

func NewUser(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	url := config.AuthCodeURL("todo", oauth2.AccessTypeOffline)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

func Publish(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	file, err := getFileByPath(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusNotFound)
		return
	}

	permission := &drive.Permission{
		Value: "",
		Type:  "anyone",
		Role:  "reader",
	}

	_, err = drive.NewPermissionsService(srv).Insert(file.Id, permission).Do()
	if err != nil {
		log.Printf("Unable to insert new permission: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	updated, err := srv.Files.Get(file.Id).Do()
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusNotFound)
		return
	}

	fmt.Fprint(w, updated.WebViewLink)
}

func Read(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	srv, err := getClient(r)
	if err != nil {
		log.Printf("Unable to retrieve drive Client: %v", err)
		http.Error(w, "Unable to parse token", http.StatusUnauthorized)
		return
	}

	file, err := getFileByPath(srv, ps.ByName("filepath"))
	if err != nil {
		log.Printf("Unable to retrieve files: %v", err)
		http.Error(w, "Unable to retrieve files", http.StatusNotFound)
		return
	}

	response, err := srv.Files.Get(file.Id).Download()
	if err != nil {
		log.Printf("Failed to get file: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	io.Copy(w, response.Body)
	response.Body.Close()
}

func Validate(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	code := r.FormValue("code")
	if code == "" {
		log.Println("No authorization code present")
		http.Error(w, "No authorization code present", http.StatusUnauthorized)
		return
	}

	token, err := config.Exchange(oauth2.NoContext, code)
	if err != nil {
		log.Printf("Unable to retrieve token: %v", err)
		http.Error(w, "Unable to retrieve token",
			http.StatusInternalServerError)
		return
	}

	fmt.Fprint(w, token.AccessToken)
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"net/http"

	"github.com/mholt/caddy/caddyhttp/httpserver"
)

// Handler is the main cache middleware
type Handler struct {
	// Handler configuration
	Config *Config

	// Cache is where entries are stored
	Cache *HTTPCache

	// Next handler
	Next httpserver.Handler

	// Handles locking for different URLs
	URLLocks *URLLock
}

const (
	cacheHit    = "hit"
	cacheMiss   = "miss"
	cacheSkip   = "skip"
	cacheBypass = "bypass"
)

func getKey(r *http.Request) string {
	key := r.Method + " " + r.Host + r.URL.Path

	q := r.URL.Query().Encode()
	if len(q) > 0 {
		key += "?" + q
	}

	return key
}

// NewHandler creates a new Handler using Next middleware
func NewHandler(Next httpserver.Handler, config *Config) *Handler {
	return &Handler{
		Config:   config,
		Cache:    NewHTTPCache(),
		URLLocks: NewURLLock(),
		Next:     Next,
	}
}

/* Responses */

func copyHeaders(from http.Header, to http.Header) {
	for k, values := range from {
		for _, v := range values {
			to.Add(k, v)
		}
	}
}

func (handler *Handler) addStatusHeaderIfConfigured(w http.ResponseWriter, status string) {
	if rec, ok := w.(*httpserver.ResponseRecorder); ok {
		rec.Replacer.Set("cache_status", status)
	}

	if handler.Config.StatusHeader != "" {
		w.Header().Add(handler.Config.StatusHeader, status)
	}
}

func (handler *Handler) respond(w http.ResponseWriter, entry *HTTPCacheEntry, cacheStatus string) (int, error) {
	handler.addStatusHeaderIfConfigured(w, cacheStatus)

	copyHeaders(entry.Response.snapHeader, w.Header())
	w.WriteHeader(entry.Response.Code)

	err := entry.WriteBodyTo(w)

	return entry.Response.Code, err
}

/* Handler */

func shouldUseCache(req *http.Request) bool {
	// TODO Add more logic like get params, ?nocache=true

	if req.Method != "GET" && req.Method != "HEAD" {
		// Only cache Get and head request
		return false
	}

	// Range requests still not supported
	// It may happen that the previous request for this url has a successful response
	// but for another Range. So a special handling is needed
	if req.Header.Get("range") != "" {
		return false
	}

	return true
}

func popOrNil(errChan chan error) (err error) {
	select {
	case err = <-errChan:
	default:
	}
	return
}

func (handler *Handler) fetchUpstream(req *http.Request) (*HTTPCacheEntry, error) {
	// Create a new empty response
	response := NewResponse()

	errChan := make(chan error, 1)

	// Do the upstream fetching in background
	go func(req *http.Request, response *Response) {
		// Create a new context to avoid terminating the Next.ServeHTTP when the original
		// request is closed. Otherwise if the original request is cancelled the other requests
		// will see a bad response that has the same contents the first request has
		updatedReq := req.WithContext(context.Background())

		statusCode, upstreamError := handler.Next.ServeHTTP(response, updatedReq)
		errChan <- upstreamError

		// If status code was not set, this will not replace it
		// It will only ensure status code IS send
		response.WriteHeader(statusCode)

		// Wait the response body to be set.
		// If it is private it will be the original http.ResponseWriter
		// It is required to wait the body to prevent closing the response
		// before the body was set. If that happens the body will
		// stay locked waiting the response to be closed
		response.WaitBody()
		response.Close()
	}(req, response)

	// Wait headers to de sent
	response.WaitHeaders()

	// Create a new CacheEntry
	return NewHTTPCacheEntry(getKey(req), req, response, handler.Config), popOrNil(errChan)
}

func (handler *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	if !shouldUseCache(r) {
		handler.addStatusHeaderIfConfigured(w, cacheBypass)
		return handler.Next.ServeHTTP(w, r)
	}

	lock := handler.URLLocks.Adquire(getKey(r))

	// Lookup correct entry
	previousEntry, exists := handler.Cache.Get(r)

	// First case: CACHE HIT
	// The response exists in cache and is public
	// It should be served as saved
	if exists && previousEntry.isPublic {
		lock.Unlock()
		return handler.respond(w, previousEntry, cacheHit)
	}

	// Second case: CACHE SKIP
	// The response is in cache but it is not public
	// It should NOT be served from cache
	// It should be fetched from upstream and check the new headers
	// To check if the new response changes to public
	if exists && !previousEntry.isPublic {
		lock.Unlock()
		entry, err := handler.fetchUpstream(r)
		if err != nil {
			return entry.Response.Code, err
		}

		// Case when response was private but now is public
		if entry.isPublic {
			err := entry.setStorage(handler.Config)
			if err != nil {
				return 500, err
			}

			handler.Cache.Put(r, entry)
			return handler.respond(w, entry, cacheMiss)
		}

		return handler.respond(w, entry, cacheSkip)
	}

	// Third case: CACHE MISS
	// The response is not in cache
	// It should be fetched from upstream and save it in cache
	entry, err := handler.fetchUpstream(r)
	if err != nil {
		lock.Unlock()
		return entry.Response.Code, err
	}

	// Entry is always saved, even if it is not public
	// This is to release the URL lock.
	if entry.isPublic {
		err := entry.setStorage(handler.Config)
		if err != nil {
			lock.Unlock()
			return 500, err
		}
	}

	handler.Cache.Put(r, entry)
	lock.Unlock()
	return handler.respond(w, entry, cacheMiss)
}
<commit_msg>Bypass Websocket connections (#18), it provents caddy from crashing. When a request is a socket handshake, the proxy middleware tries to hijack the connection, but the http.ResponseWritter is not the original TCP connection anymore.<commit_after>package cache

import (
	"context"
	"net/http"
	"strings"

	"github.com/mholt/caddy/caddyhttp/httpserver"
)

// Handler is the main cache middleware
type Handler struct {
	// Handler configuration
	Config *Config

	// Cache is where entries are stored
	Cache *HTTPCache

	// Next handler
	Next httpserver.Handler

	// Handles locking for different URLs
	URLLocks *URLLock
}

const (
	cacheHit    = "hit"
	cacheMiss   = "miss"
	cacheSkip   = "skip"
	cacheBypass = "bypass"
)

func getKey(r *http.Request) string {
	key := r.Method + " " + r.Host + r.URL.Path

	q := r.URL.Query().Encode()
	if len(q) > 0 {
		key += "?" + q
	}

	return key
}

// NewHandler creates a new Handler using Next middleware
func NewHandler(Next httpserver.Handler, config *Config) *Handler {
	return &Handler{
		Config:   config,
		Cache:    NewHTTPCache(),
		URLLocks: NewURLLock(),
		Next:     Next,
	}
}

/* Responses */

func copyHeaders(from http.Header, to http.Header) {
	for k, values := range from {
		for _, v := range values {
			to.Add(k, v)
		}
	}
}

func (handler *Handler) addStatusHeaderIfConfigured(w http.ResponseWriter, status string) {
	if rec, ok := w.(*httpserver.ResponseRecorder); ok {
		rec.Replacer.Set("cache_status", status)
	}

	if handler.Config.StatusHeader != "" {
		w.Header().Add(handler.Config.StatusHeader, status)
	}
}

func (handler *Handler) respond(w http.ResponseWriter, entry *HTTPCacheEntry, cacheStatus string) (int, error) {
	handler.addStatusHeaderIfConfigured(w, cacheStatus)

	copyHeaders(entry.Response.snapHeader, w.Header())
	w.WriteHeader(entry.Response.Code)

	err := entry.WriteBodyTo(w)

	return entry.Response.Code, err
}

/* Handler */

func shouldUseCache(req *http.Request) bool {
	// TODO Add more logic like get params, ?nocache=true

	if req.Method != "GET" && req.Method != "HEAD" {
		// Only cache Get and head request
		return false
	}

	// Range requests still not supported
	// It may happen that the previous request for this url has a successful response
	// but for another Range. So a special handling is needed
	if req.Header.Get("range") != "" {
		return false
	}

	if strings.ToLower(req.Header.Get("Connection")) == "upgrade" && strings.ToLower(req.Header.Get("Upgrade")) == "websocket" {
		return false
	}

	return true
}

func popOrNil(errChan chan error) (err error) {
	select {
	case err = <-errChan:
	default:
	}
	return
}

func (handler *Handler) fetchUpstream(req *http.Request) (*HTTPCacheEntry, error) {
	// Create a new empty response
	response := NewResponse()

	errChan := make(chan error, 1)

	// Do the upstream fetching in background
	go func(req *http.Request, response *Response) {
		// Create a new context to avoid terminating the Next.ServeHTTP when the original
		// request is closed. Otherwise if the original request is cancelled the other requests
		// will see a bad response that has the same contents the first request has
		updatedReq := req.WithContext(context.Background())

		statusCode, upstreamError := handler.Next.ServeHTTP(response, updatedReq)
		errChan <- upstreamError

		// If status code was not set, this will not replace it
		// It will only ensure status code IS send
		response.WriteHeader(statusCode)

		// Wait the response body to be set.
		// If it is private it will be the original http.ResponseWriter
		// It is required to wait the body to prevent closing the response
		// before the body was set. If that happens the body will
		// stay locked waiting the response to be closed
		response.WaitBody()
		response.Close()
	}(req, response)

	// Wait headers to de sent
	response.WaitHeaders()

	// Create a new CacheEntry
	return NewHTTPCacheEntry(getKey(req), req, response, handler.Config), popOrNil(errChan)
}

func (handler *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	if !shouldUseCache(r) {
		handler.addStatusHeaderIfConfigured(w, cacheBypass)
		return handler.Next.ServeHTTP(w, r)
	}

	lock := handler.URLLocks.Adquire(getKey(r))

	// Lookup correct entry
	previousEntry, exists := handler.Cache.Get(r)

	// First case: CACHE HIT
	// The response exists in cache and is public
	// It should be served as saved
	if exists && previousEntry.isPublic {
		lock.Unlock()
		return handler.respond(w, previousEntry, cacheHit)
	}

	// Second case: CACHE SKIP
	// The response is in cache but it is not public
	// It should NOT be served from cache
	// It should be fetched from upstream and check the new headers
	// To check if the new response changes to public
	if exists && !previousEntry.isPublic {
		lock.Unlock()
		entry, err := handler.fetchUpstream(r)
		if err != nil {
			return entry.Response.Code, err
		}

		// Case when response was private but now is public
		if entry.isPublic {
			err := entry.setStorage(handler.Config)
			if err != nil {
				return 500, err
			}

			handler.Cache.Put(r, entry)
			return handler.respond(w, entry, cacheMiss)
		}

		return handler.respond(w, entry, cacheSkip)
	}

	// Third case: CACHE MISS
	// The response is not in cache
	// It should be fetched from upstream and save it in cache
	entry, err := handler.fetchUpstream(r)
	if err != nil {
		lock.Unlock()
		return entry.Response.Code, err
	}

	// Entry is always saved, even if it is not public
	// This is to release the URL lock.
	if entry.isPublic {
		err := entry.setStorage(handler.Config)
		if err != nil {
			lock.Unlock()
			return 500, err
		}
	}

	handler.Cache.Put(r, entry)
	lock.Unlock()
	return handler.respond(w, entry, cacheMiss)
}
<|endoftext|>


<commit_before>package logging

import (
	"bytes"
	"io"
	"os"
	"sync"
	"text/template"
)

const (
	DefaultTimeLayout = "2006-01-02 15:04:05"
	DefaultFormat     = "[{{.TimeString}}] {{.Level}} {{.Message}}\n"
)

var (
	FileCreateFlag             = os.O_CREATE | os.O_APPEND | os.O_WRONLY
	FileCreatePerm os.FileMode = 0640
)

type Handler struct {
	mutex      sync.Mutex
	buffer     *bytes.Buffer
	writer     io.Writer
	level      logLevel
	lRange     *levelRange
	timeLayout string
	tmpl       *template.Template
	filter     func(*Record) bool

	Before func(*Record, io.ReadWriter)
	After  func(*Record, int64)
}

func NewHandler(out io.Writer, level logLevel, layout, format string) (*Handler, error) {
	h := &Handler{
		buffer:     bytes.NewBuffer(nil),
		writer:     out,
		level:      level,
		timeLayout: layout,
	}
	if err := h.SetFormat(format); err != nil {
		return nil, err
	}
	return h, nil
}

func (h *Handler) SetLevel(level logLevel) {
	h.level = level
}

func (h *Handler) SetLevelString(s string) {
	h.SetLevel(stringToLogLevel(s))
}

func (h *Handler) SetLevelRange(minLevel, maxLevel logLevel) {
	h.lRange = &levelRange{minLevel, maxLevel}
}

func (h *Handler) SetLevelRangeString(smin, smax string) {
	h.SetLevelRange(stringToLogLevel(smin), stringToLogLevel(smax))
}

func (h *Handler) SetTimeLayout(layout string) {
	h.timeLayout = layout
}

func (h *Handler) SetFormat(format string) error {
	tmpl, err := template.New("tmpl").Parse(format)
	if err != nil {
		return err
	}
	h.tmpl = tmpl
	return nil
}

func (h *Handler) SetFilter(f func(*Record) bool) {
	h.filter = f
}

func (h *Handler) Emit(rd Record) {
	if h.lRange != nil {
		if !h.lRange.contains(rd.Level) {
			return
		}
	} else if h.level > rd.Level {
		return
	}
	h.mutex.Lock()
	h.buffer.Reset()
	h.handleRecord(&rd, h.buffer)
	h.mutex.Unlock()
}

func (h *Handler) handleRecord(rd *Record, buf *bytes.Buffer) {
	if h.writer == nil {
		return
	}
	if h.filter != nil && h.filter(rd) {
		return
	}
	rd.TimeString = rd.Time.Format(h.timeLayout)
	if err := h.tmpl.Execute(buf, rd); err != nil {
		return
	}
	if h.Before != nil {
		h.Before(rd, buf)
	}
	n, err := io.Copy(h.writer, buf)
	if err != nil {
		return
	}
	if h.After != nil {
		h.After(rd, int64(n))
	}
}
<commit_msg>more find-grained write buffer lock<commit_after>package logging

import (
	"bytes"
	"io"
	"os"
	"sync"
	"text/template"
)

const (
	DefaultTimeLayout = "2006-01-02 15:04:05"
	DefaultFormat     = "[{{.TimeString}}] {{.Level}} {{.Message}}\n"
)

var (
	FileCreateFlag             = os.O_CREATE | os.O_APPEND | os.O_WRONLY
	FileCreatePerm os.FileMode = 0640
)

type Handler struct {
	mutex      sync.Mutex
	buffer     *bytes.Buffer
	writer     io.Writer
	level      logLevel
	lRange     *levelRange
	timeLayout string
	tmpl       *template.Template
	filter     func(*Record) bool

	Before func(*Record, io.ReadWriter)
	After  func(*Record, int64)
}

func NewHandler(out io.Writer, level logLevel, layout, format string) (*Handler, error) {
	h := &Handler{
		buffer:     bytes.NewBuffer(nil),
		writer:     out,
		level:      level,
		timeLayout: layout,
	}
	if err := h.SetFormat(format); err != nil {
		return nil, err
	}
	return h, nil
}

func (h *Handler) SetLevel(level logLevel) {
	h.level = level
}

func (h *Handler) SetLevelString(s string) {
	h.SetLevel(stringToLogLevel(s))
}

func (h *Handler) SetLevelRange(minLevel, maxLevel logLevel) {
	h.lRange = &levelRange{minLevel, maxLevel}
}

func (h *Handler) SetLevelRangeString(smin, smax string) {
	h.SetLevelRange(stringToLogLevel(smin), stringToLogLevel(smax))
}

func (h *Handler) SetTimeLayout(layout string) {
	h.timeLayout = layout
}

func (h *Handler) SetFormat(format string) error {
	tmpl, err := template.New("tmpl").Parse(format)
	if err != nil {
		return err
	}
	h.tmpl = tmpl
	return nil
}

func (h *Handler) SetFilter(f func(*Record) bool) {
	h.filter = f
}

func (h *Handler) Emit(rd Record) {
	if h.lRange != nil {
		if !h.lRange.contains(rd.Level) {
			return
		}
	} else if h.level > rd.Level {
		return
	}
	h.handleRecord(&rd)
}

func (h *Handler) handleRecord(rd *Record) {
	if h.writer == nil {
		return
	}
	if h.filter != nil && h.filter(rd) {
		return
	}
	rd.TimeString = rd.Time.Format(h.timeLayout)
	h.mutex.Lock()
	h.buffer.Reset()
	if err := h.tmpl.Execute(h.buffer, rd); err != nil {
		h.mutex.Unlock()
		return
	}
	if h.Before != nil {
		h.Before(rd, h.buffer)
	}
	n, err := io.Copy(h.writer, h.buffer)
	if err != nil {
		h.mutex.Unlock()
		return
	}
	h.mutex.Unlock()
	if h.After != nil {
		h.After(rd, int64(n))
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2016, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package cmd

import (
	"errors"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"time"
)

var (
	DefaultTimeout = 60 * time.Second
)

var (
	ErrNotFound                = errors.New("Executable file not found in $PATH")
	ErrTimeout                 = errors.New("Timeout")
	ErrKillProcessAfterTimeout = errors.New("Failed to kill process after timeout")
)

// Wrap os/exec/Cmd so we can test commands.
type Cmd interface {
	Run() (output string, err error)
}

type CmdFactory interface {
	Make(name string, args ...string) Cmd
}

// Set in main/percona-agent/main.go to RealCmdFactory for real agent,
// else set in tests to mock.CmdFactory for testing.
var Factory CmdFactory

// --------------------------------------------------------------------------

type RealCmdFactory struct {
}

func (f *RealCmdFactory) Make(name string, args ...string) Cmd {
	return NewRealCmd(name, args...)
}

type RealCmd struct {
	Timeout time.Duration
	name    string
	args    []string
}

type result struct {
	output string
	err    error
}

func NewRealCmd(name string, args ...string) *RealCmd {
	return &RealCmd{
		name:    name,
		args:    args,
		Timeout: DefaultTimeout,
	}
}

func (c *RealCmd) Run() (output string, err error) {
	var basepath string
	if binfile, err := os.Executable(); err != nil {
		basepath = path.Dir(binfile)
		osPath := os.Getenv("PATH")
		_ = os.Setenv("PATH", basepath+"/bin/"+string(filepath.ListSeparator)+osPath)
	}
	cmd := exec.Command(c.name, c.args...)

	// Workaround for "HOME: parameter not set"
	if os.Getenv("HOME") == "" {
		cmd.Env = append(os.Environ(), "HOME=/root")
	}

	resultChan := runCmd(cmd)
	// Restore the path if it was changed
	if basepath != "" {
		os.Setenv("PATH", basepath)
	}
	select {
	case <-time.After(c.Timeout):
		killErr := cmd.Process.Kill()
		if killErr != nil {
			// @todo:
			// If this happens that means leaving working process,
			// plus working goroutine waiting for that process to finish.
			// And since this command is going to be run over, and over again
			// we might end up with hundreds processes and goroutines hanging.
			// Maybe in such critical cases (or after n-cases) we should shutdown whole module (e.g. qan/mm/summary)
			// and notify us (developers), because this shouldn't happen in correct working program - but you never know
			return "", ErrKillProcessAfterTimeout
		}
		return "", ErrTimeout
	case result := <-resultChan:
		execError, ok := result.err.(*exec.Error)
		if ok && execError.Err == exec.ErrNotFound {
			return "", ErrNotFound
		}
		return result.output, result.err
	}
}

func runCmd(cmd *exec.Cmd) (resultChan chan result) {
	// Below channels has buffer
	// because we might get data before we would be waiting on this channel
	resultChan = make(chan result, 1)
	go func() {
		output, err := cmd.CombinedOutput()
		select {
		case resultChan <- result{output: string(output), err: err}:
		default:
		}
	}()
	return resultChan
}
<commit_msg>PMM-873 removed uneeded assignment to _<commit_after>/*
   Copyright (c) 2016, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package cmd

import (
	"errors"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"time"
)

var (
	DefaultTimeout = 60 * time.Second
)

var (
	ErrNotFound                = errors.New("Executable file not found in $PATH")
	ErrTimeout                 = errors.New("Timeout")
	ErrKillProcessAfterTimeout = errors.New("Failed to kill process after timeout")
)

// Wrap os/exec/Cmd so we can test commands.
type Cmd interface {
	Run() (output string, err error)
}

type CmdFactory interface {
	Make(name string, args ...string) Cmd
}

// Set in main/percona-agent/main.go to RealCmdFactory for real agent,
// else set in tests to mock.CmdFactory for testing.
var Factory CmdFactory

// --------------------------------------------------------------------------

type RealCmdFactory struct {
}

func (f *RealCmdFactory) Make(name string, args ...string) Cmd {
	return NewRealCmd(name, args...)
}

type RealCmd struct {
	Timeout time.Duration
	name    string
	args    []string
}

type result struct {
	output string
	err    error
}

func NewRealCmd(name string, args ...string) *RealCmd {
	return &RealCmd{
		name:    name,
		args:    args,
		Timeout: DefaultTimeout,
	}
}

func (c *RealCmd) Run() (output string, err error) {
	var basepath string
	if binfile, err := os.Executable(); err != nil {
		basepath = path.Dir(binfile)
		osPath := os.Getenv("PATH")
		os.Setenv("PATH", basepath+"/bin/"+string(filepath.ListSeparator)+osPath)
	}
	cmd := exec.Command(c.name, c.args...)

	// Workaround for "HOME: parameter not set"
	if os.Getenv("HOME") == "" {
		cmd.Env = append(os.Environ(), "HOME=/root")
	}

	resultChan := runCmd(cmd)
	// Restore the path if it was changed
	if basepath != "" {
		os.Setenv("PATH", basepath)
	}
	select {
	case <-time.After(c.Timeout):
		killErr := cmd.Process.Kill()
		if killErr != nil {
			// @todo:
			// If this happens that means leaving working process,
			// plus working goroutine waiting for that process to finish.
			// And since this command is going to be run over, and over again
			// we might end up with hundreds processes and goroutines hanging.
			// Maybe in such critical cases (or after n-cases) we should shutdown whole module (e.g. qan/mm/summary)
			// and notify us (developers), because this shouldn't happen in correct working program - but you never know
			return "", ErrKillProcessAfterTimeout
		}
		return "", ErrTimeout
	case result := <-resultChan:
		execError, ok := result.err.(*exec.Error)
		if ok && execError.Err == exec.ErrNotFound {
			return "", ErrNotFound
		}
		return result.output, result.err
	}
}

func runCmd(cmd *exec.Cmd) (resultChan chan result) {
	// Below channels has buffer
	// because we might get data before we would be waiting on this channel
	resultChan = make(chan result, 1)
	go func() {
		output, err := cmd.CombinedOutput()
		select {
		case resultChan <- result{output: string(output), err: err}:
		default:
		}
	}()
	return resultChan
}
<|endoftext|>


<commit_before>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

// +build ignore

// xflate_stats is used to investigate trade-offs in XFLATE design.
//
// The XFLATE format extends the DEFLATE format in order to allow for limited
// random access reading. This is achieved by individually compressing the
// input data in chunks (so that each chunk can be decompressed by itself),
// and by storing an index recording the size of every chunk (so that each
// chunk can be located).
//
// However, this adds overhead and diminishes the compression ratio slightly.
// In order to investigate the cost of chunking, this program computes the total
// compressed size when the input is compressed as a single stream or when
// compressed in chunks of some fixed size.
//
// Chunking diminishes compression ratio because the LZ77 dictionary is reset
// for each chunk, reducing the amount of savings that could have been gained
// by a backwards match to previous data. Secondly, the XFLATE format requires
// that each chunk terminates with a SYNC_FLUSH marker of ~5 bytes.
//
// In addition to the costs of chunking itself, there is also the cost of
// storing the index. The index is a list of (rawSize, compSize) tuples where
// each record contains information about the uncompressed input size (rawSize)
// and compressed output size (compSize) of each chunk.
//
// In order to reduce the index size, multiple techniques were explored:
//	* Compressing the index itself using DEFLATE
//	* Row-oriented vs. column-oriented layout of records; that is, row-oriented
//	layout has each (rawSize, compSize) tuple laid out one after the other,
//	while column-oriented has all rawSizes laid out as an array followed by all
//	compSizes laid out as an array
//	* Fixed-length vs. variable-length integers; that is, to store size values
//	as uint64s or to use some variable-length encoding
//	* Regular encoding vs. Delta encoding; that is, delta encoding encodes each
//	value as the difference between that value and the previous value
//
// However, some of these techniques do not make sense unless done in
// conjunction with another technique. For example:
//	* Row-oriented vs. column-oriented is useless without compression
//	* Delta encoding does not make sense without variable-length integers
//
// For this reason, the following table shows what techniques were used for the
// given format names listed below:
//
//	         +-------------- Raw: The index is uncompressed
//	         |   +---------- Row: The index is row-oriented
//	         |   |   +------ Fix: Index records use fixed-length uint64s
//	         |   |   |   +-- Reg: Index records are not delta encoded
//	         |   |   |   |
//	        Raw Row Fix Reg
//	RawFix  [X] [?] [X] [X]
//	RawVar  [X] [?] [ ] [X]
//	RawDlt  [X] [?] [ ] [ ]
//	RowFix  [ ] [X] [X] [X]
//	RowVar  [ ] [X] [ ] [X]
//	RowDlt  [ ] [X] [ ] [ ]
//	ColFix  [ ] [ ] [X] [X]
//	ColVar  [ ] [ ] [ ] [X]
//	ColDlt  [ ] [ ] [ ] [ ]
//
// As seen, the Raw* formats do not care about the orientation of the index
// since it is not compressed. Also, all *Dlt formats uses delta encoding
// in conjunction with variable-length encoding.
//
// In summary, the greatest factor for overhead cost is the chunk size.
// Smaller chunks reduce efficiency due to more frequent LZ77 dictionary resets
// and also increases the total number of chunks needed, thus also increasing
// the index size. On the other hand, larger chunks also diminishes the
// effectiveness of random access reading since more data must be discarded to
// start reading from some given offset. Thus, the choice of chunk size is not
// hard-coded by the XFLATE format and this tool can help identify the
// appropriate chunk size.
package main

import (
	"bytes"
	"compress/flate"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/dsnet/compress/xflate/meta"
)

func init() { log.SetFlags(log.Lshortfile) }

func main() {
	inputFile := flag.String("f", "-", "path to input file")
	chunkSize := flag.Int("n", 64*1024, "compress the input in n-sized chunks")
	compLevel := flag.Int("l", flate.DefaultCompression, "compression level")
	metaEncode := flag.Bool("m", false, "encode index using XFLATE meta encoding")
	flag.Parse()

	// Open the input file.
	var r io.Reader
	if *inputFile != "-" {
		f, err := os.Open(*inputFile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		r = f
	} else {
		r = os.Stdin
	}

	// Compute the streamed and chunked record values.
	var chnkRec indexRecord
	strmRec, chnkRecs := computeRecords(r, *compLevel, *chunkSize)
	for _, r := range chnkRecs {
		chnkRec.rawSize += r.rawSize
		chnkRec.compSize += r.compSize
	}

	var n int
	var pr indexRecord // Previous record
	var brf, brv, brd, bcf, bcv, bcd bytes.Buffer
	buf := make([]byte, binary.MaxVarintLen64)

	// Row-based index format.
	pr = indexRecord{}
	for _, r := range chnkRecs {
		binary.LittleEndian.PutUint64(buf, uint64(r.rawSize))
		brf.Write(buf[:8])
		binary.LittleEndian.PutUint64(buf, uint64(r.compSize))
		brf.Write(buf[:8])

		n = binary.PutUvarint(buf, uint64(r.rawSize))
		brv.Write(buf[:n])
		n = binary.PutUvarint(buf, uint64(r.compSize))
		brv.Write(buf[:n])

		n = binary.PutVarint(buf, r.rawSize-pr.rawSize)
		brd.Write(buf[:n])
		n = binary.PutVarint(buf, r.compSize-pr.compSize)
		brd.Write(buf[:n])

		pr = r
	}

	// Column-based index format.
	pr = indexRecord{}
	for _, r := range chnkRecs {
		binary.LittleEndian.PutUint64(buf, uint64(r.rawSize))
		bcf.Write(buf[:8])

		n = binary.PutUvarint(buf, uint64(r.rawSize))
		bcv.Write(buf[:n])

		n = binary.PutVarint(buf, r.rawSize-pr.rawSize)
		bcd.Write(buf[:n])

		pr.rawSize = r.rawSize
	}
	for _, r := range chnkRecs {
		binary.LittleEndian.PutUint64(buf, uint64(r.compSize))
		bcf.Write(buf[:8])

		n = binary.PutUvarint(buf, uint64(r.compSize))
		bcv.Write(buf[:n])

		n = binary.PutVarint(buf, r.compSize-pr.compSize)
		bcd.Write(buf[:n])

		pr.compSize = r.compSize
	}

	pf := func(a, b int64) float64 { return 100.0 * (float64(a) / float64(b)) }
	me := func(b []byte) []byte { return b }
	if *metaEncode {
		me = encode
	}

	// Print basic statistics about the input and output.
	ns, nc := strmRec.compSize, chnkRec.compSize
	ps, pc := pf(ns-ns, ns), pf(nc-ns, ns)
	fmt.Printf("rawSize:          %d\n", strmRec.rawSize)   // Uncompressed input size
	fmt.Printf("strmRec.compSize: %d (%+0.2f%%)\n", ns, ps) // Total compressed size as a single stream
	fmt.Printf("chnkRec.compSize: %d (%+0.2f%%)\n", nc, pc) // Total compressed size as individual chunks
	fmt.Printf("chunkSize:        %d\n", *chunkSize)        // Individual chunk size
	fmt.Printf("numChunks:        %d\n", len(chnkRecs))     // Total number of chunks
	fmt.Println()

	// Uncompressed index formats.
	nf := int64(len(me(brf.Bytes())))
	nv := int64(len(me(brv.Bytes())))
	nd := int64(len(me(brd.Bytes())))
	fmt.Printf("RawFix: %d (%+0.2f%%)\n", nf, pf(nf, ns)) // Fixed-length uint64s
	fmt.Printf("RawVar: %d (%+0.2f%%)\n", nv, pf(nv, ns)) // Variable-length integers (VLI)
	fmt.Printf("RawDlt: %d (%+0.2f%%)\n", nd, pf(nd, ns)) // VLI with delta encoding
	fmt.Println()

	// Compressed row-oriented index formats.
	nrf := int64(len(me(compress(brf.Bytes(), *compLevel))))
	nrv := int64(len(me(compress(brv.Bytes(), *compLevel))))
	nrd := int64(len(me(compress(brd.Bytes(), *compLevel))))
	fmt.Printf("RowFix: %d (%+0.2f%%)\n", nrf, pf(nrf, ns)) // Fixed-length uint64s
	fmt.Printf("RowVar: %d (%+0.2f%%)\n", nrv, pf(nrv, ns)) // Variable-length integers (VLI)
	fmt.Printf("RowDlt: %d (%+0.2f%%)\n", nrd, pf(nrd, ns)) // VLI with delta encoding
	fmt.Println()

	// Compressed column-oriented index formats.
	ncf := int64(len(me(compress(bcf.Bytes(), *compLevel))))
	ncv := int64(len(me(compress(bcv.Bytes(), *compLevel))))
	ncd := int64(len(me(compress(bcd.Bytes(), *compLevel))))
	fmt.Printf("ColFix: %d (%+0.2f%%)\n", ncf, pf(ncf, ns)) // Fixed-length uint64s
	fmt.Printf("ColVar: %d (%+0.2f%%)\n", ncv, pf(ncv, ns)) // Variable-length integers (VLI)
	fmt.Printf("ColDlt: %d (%+0.2f%%)\n", ncd, pf(ncd, ns)) // VLI with delta encoding
	fmt.Println()
}

// countWriter counts and discards all bytes written to it.
type countWriter int64

func (cw *countWriter) Write(b []byte) (int, error) {
	*(*int64)(cw) += int64(len(b))
	return len(b), nil
}

type indexRecord struct {
	rawSize  int64 // Size when decompressed
	compSize int64 // Size when compressed
}

// computeRecords computes the records the raw input size and the compressed
// output size. strmRec is a single record for when the input is compressed as
// a single stream. chnkRecs is a list of records for when the input is
// compressed individually as chunks.
func computeRecords(r io.Reader, lvl, chnkSize int) (strmRec indexRecord, chnkRecs []indexRecord) {
	var cw1, cw2 countWriter
	zw1, _ := flate.NewWriter(&cw1, lvl) // Streamed compressor
	zw2, _ := flate.NewWriter(&cw2, lvl) // Chunked compressor
	buf := make([]byte, chnkSize)
	for {
		// Read a full chunks worth of data.
		cnt, err := io.ReadFull(r, buf)
		strmRec.rawSize += int64(cnt)
		if err == io.EOF {
			break
		}

		// Write chunk to both compressors.
		if _, err := zw1.Write(buf[:cnt]); err != nil {
			log.Fatal(err)
		}
		if _, err := zw2.Write(buf[:cnt]); err != nil {
			log.Fatal(err)
		}

		// Flush the chunked compressor, append the record, and reset.
		if err := zw2.Flush(); err != nil {
			log.Fatal(err)
		}
		chnkRecs = append(chnkRecs, indexRecord{rawSize: int64(cnt), compSize: int64(cw2)})
		cw2 = 0
		zw2.Reset(&cw2)

		if err == io.ErrUnexpectedEOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
	}

	// Flush the streamed compressor and record the compressed size.
	if err := zw1.Flush(); err != nil {
		log.Fatal(err)
	}
	strmRec.compSize = int64(cw1)
	return strmRec, chnkRecs
}

// compress compresses the input buffer at the given level.
func compress(b []byte, lvl int) []byte {
	var buf bytes.Buffer
	w, err := flate.NewWriter(&buf, lvl)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := io.Copy(w, bytes.NewReader(b)); err != nil {
		log.Fatal(err)
	}
	if err := w.Close(); err != nil {
		log.Fatal(err)
	}
	return buf.Bytes()
}

// encode encodes the input using XFLATE's meta encoding.
func encode(b []byte) []byte {
	var buf bytes.Buffer
	mw := meta.NewWriter(&buf)
	mw.FinalMode = meta.FinalMeta
	if _, err := io.Copy(mw, bytes.NewReader(b)); err != nil {
		log.Fatal(err)
	}
	if err := mw.Close(); err != nil {
		log.Fatal(err)
	}
	return buf.Bytes()
}
<commit_msg>xflate: update import in xflate_stats.go<commit_after>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

// +build ignore

// xflate_stats is used to investigate trade-offs in XFLATE design.
//
// The XFLATE format extends the DEFLATE format in order to allow for limited
// random access reading. This is achieved by individually compressing the
// input data in chunks (so that each chunk can be decompressed by itself),
// and by storing an index recording the size of every chunk (so that each
// chunk can be located).
//
// However, this adds overhead and diminishes the compression ratio slightly.
// In order to investigate the cost of chunking, this program computes the total
// compressed size when the input is compressed as a single stream or when
// compressed in chunks of some fixed size.
//
// Chunking diminishes compression ratio because the LZ77 dictionary is reset
// for each chunk, reducing the amount of savings that could have been gained
// by a backwards match to previous data. Secondly, the XFLATE format requires
// that each chunk terminates with a SYNC_FLUSH marker of ~5 bytes.
//
// In addition to the costs of chunking itself, there is also the cost of
// storing the index. The index is a list of (rawSize, compSize) tuples where
// each record contains information about the uncompressed input size (rawSize)
// and compressed output size (compSize) of each chunk.
//
// In order to reduce the index size, multiple techniques were explored:
//	* Compressing the index itself using DEFLATE
//	* Row-oriented vs. column-oriented layout of records; that is, row-oriented
//	layout has each (rawSize, compSize) tuple laid out one after the other,
//	while column-oriented has all rawSizes laid out as an array followed by all
//	compSizes laid out as an array
//	* Fixed-length vs. variable-length integers; that is, to store size values
//	as uint64s or to use some variable-length encoding
//	* Regular encoding vs. Delta encoding; that is, delta encoding encodes each
//	value as the difference between that value and the previous value
//
// However, some of these techniques do not make sense unless done in
// conjunction with another technique. For example:
//	* Row-oriented vs. column-oriented is useless without compression
//	* Delta encoding does not make sense without variable-length integers
//
// For this reason, the following table shows what techniques were used for the
// given format names listed below:
//
//	         +-------------- Raw: The index is uncompressed
//	         |   +---------- Row: The index is row-oriented
//	         |   |   +------ Fix: Index records use fixed-length uint64s
//	         |   |   |   +-- Reg: Index records are not delta encoded
//	         |   |   |   |
//	        Raw Row Fix Reg
//	RawFix  [X] [?] [X] [X]
//	RawVar  [X] [?] [ ] [X]
//	RawDlt  [X] [?] [ ] [ ]
//	RowFix  [ ] [X] [X] [X]
//	RowVar  [ ] [X] [ ] [X]
//	RowDlt  [ ] [X] [ ] [ ]
//	ColFix  [ ] [ ] [X] [X]
//	ColVar  [ ] [ ] [ ] [X]
//	ColDlt  [ ] [ ] [ ] [ ]
//
// As seen, the Raw* formats do not care about the orientation of the index
// since it is not compressed. Also, all *Dlt formats uses delta encoding
// in conjunction with variable-length encoding.
//
// In summary, the greatest factor for overhead cost is the chunk size.
// Smaller chunks reduce efficiency due to more frequent LZ77 dictionary resets
// and also increases the total number of chunks needed, thus also increasing
// the index size. On the other hand, larger chunks also diminishes the
// effectiveness of random access reading since more data must be discarded to
// start reading from some given offset. Thus, the choice of chunk size is not
// hard-coded by the XFLATE format and this tool can help identify the
// appropriate chunk size.
package main

import (
	"bytes"
	"compress/flate"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/dsnet/compress/xflate/internal/meta"
)

func init() { log.SetFlags(log.Lshortfile) }

func main() {
	inputFile := flag.String("f", "-", "path to input file")
	chunkSize := flag.Int("n", 64*1024, "compress the input in n-sized chunks")
	compLevel := flag.Int("l", flate.DefaultCompression, "compression level")
	metaEncode := flag.Bool("m", false, "encode index using XFLATE meta encoding")
	flag.Parse()

	// Open the input file.
	var r io.Reader
	if *inputFile != "-" {
		f, err := os.Open(*inputFile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		r = f
	} else {
		r = os.Stdin
	}

	// Compute the streamed and chunked record values.
	var chnkRec indexRecord
	strmRec, chnkRecs := computeRecords(r, *compLevel, *chunkSize)
	for _, r := range chnkRecs {
		chnkRec.rawSize += r.rawSize
		chnkRec.compSize += r.compSize
	}

	var n int
	var pr indexRecord // Previous record
	var brf, brv, brd, bcf, bcv, bcd bytes.Buffer
	buf := make([]byte, binary.MaxVarintLen64)

	// Row-based index format.
	pr = indexRecord{}
	for _, r := range chnkRecs {
		binary.LittleEndian.PutUint64(buf, uint64(r.rawSize))
		brf.Write(buf[:8])
		binary.LittleEndian.PutUint64(buf, uint64(r.compSize))
		brf.Write(buf[:8])

		n = binary.PutUvarint(buf, uint64(r.rawSize))
		brv.Write(buf[:n])
		n = binary.PutUvarint(buf, uint64(r.compSize))
		brv.Write(buf[:n])

		n = binary.PutVarint(buf, r.rawSize-pr.rawSize)
		brd.Write(buf[:n])
		n = binary.PutVarint(buf, r.compSize-pr.compSize)
		brd.Write(buf[:n])

		pr = r
	}

	// Column-based index format.
	pr = indexRecord{}
	for _, r := range chnkRecs {
		binary.LittleEndian.PutUint64(buf, uint64(r.rawSize))
		bcf.Write(buf[:8])

		n = binary.PutUvarint(buf, uint64(r.rawSize))
		bcv.Write(buf[:n])

		n = binary.PutVarint(buf, r.rawSize-pr.rawSize)
		bcd.Write(buf[:n])

		pr.rawSize = r.rawSize
	}
	for _, r := range chnkRecs {
		binary.LittleEndian.PutUint64(buf, uint64(r.compSize))
		bcf.Write(buf[:8])

		n = binary.PutUvarint(buf, uint64(r.compSize))
		bcv.Write(buf[:n])

		n = binary.PutVarint(buf, r.compSize-pr.compSize)
		bcd.Write(buf[:n])

		pr.compSize = r.compSize
	}

	pf := func(a, b int64) float64 { return 100.0 * (float64(a) / float64(b)) }
	me := func(b []byte) []byte { return b }
	if *metaEncode {
		me = encode
	}

	// Print basic statistics about the input and output.
	ns, nc := strmRec.compSize, chnkRec.compSize
	ps, pc := pf(ns-ns, ns), pf(nc-ns, ns)
	fmt.Printf("rawSize:          %d\n", strmRec.rawSize)   // Uncompressed input size
	fmt.Printf("strmRec.compSize: %d (%+0.2f%%)\n", ns, ps) // Total compressed size as a single stream
	fmt.Printf("chnkRec.compSize: %d (%+0.2f%%)\n", nc, pc) // Total compressed size as individual chunks
	fmt.Printf("chunkSize:        %d\n", *chunkSize)        // Individual chunk size
	fmt.Printf("numChunks:        %d\n", len(chnkRecs))     // Total number of chunks
	fmt.Println()

	// Uncompressed index formats.
	nf := int64(len(me(brf.Bytes())))
	nv := int64(len(me(brv.Bytes())))
	nd := int64(len(me(brd.Bytes())))
	fmt.Printf("RawFix: %d (%+0.2f%%)\n", nf, pf(nf, ns)) // Fixed-length uint64s
	fmt.Printf("RawVar: %d (%+0.2f%%)\n", nv, pf(nv, ns)) // Variable-length integers (VLI)
	fmt.Printf("RawDlt: %d (%+0.2f%%)\n", nd, pf(nd, ns)) // VLI with delta encoding
	fmt.Println()

	// Compressed row-oriented index formats.
	nrf := int64(len(me(compress(brf.Bytes(), *compLevel))))
	nrv := int64(len(me(compress(brv.Bytes(), *compLevel))))
	nrd := int64(len(me(compress(brd.Bytes(), *compLevel))))
	fmt.Printf("RowFix: %d (%+0.2f%%)\n", nrf, pf(nrf, ns)) // Fixed-length uint64s
	fmt.Printf("RowVar: %d (%+0.2f%%)\n", nrv, pf(nrv, ns)) // Variable-length integers (VLI)
	fmt.Printf("RowDlt: %d (%+0.2f%%)\n", nrd, pf(nrd, ns)) // VLI with delta encoding
	fmt.Println()

	// Compressed column-oriented index formats.
	ncf := int64(len(me(compress(bcf.Bytes(), *compLevel))))
	ncv := int64(len(me(compress(bcv.Bytes(), *compLevel))))
	ncd := int64(len(me(compress(bcd.Bytes(), *compLevel))))
	fmt.Printf("ColFix: %d (%+0.2f%%)\n", ncf, pf(ncf, ns)) // Fixed-length uint64s
	fmt.Printf("ColVar: %d (%+0.2f%%)\n", ncv, pf(ncv, ns)) // Variable-length integers (VLI)
	fmt.Printf("ColDlt: %d (%+0.2f%%)\n", ncd, pf(ncd, ns)) // VLI with delta encoding
	fmt.Println()
}

// countWriter counts and discards all bytes written to it.
type countWriter int64

func (cw *countWriter) Write(b []byte) (int, error) {
	*(*int64)(cw) += int64(len(b))
	return len(b), nil
}

type indexRecord struct {
	rawSize  int64 // Size when decompressed
	compSize int64 // Size when compressed
}

// computeRecords computes the records the raw input size and the compressed
// output size. strmRec is a single record for when the input is compressed as
// a single stream. chnkRecs is a list of records for when the input is
// compressed individually as chunks.
func computeRecords(r io.Reader, lvl, chnkSize int) (strmRec indexRecord, chnkRecs []indexRecord) {
	var cw1, cw2 countWriter
	zw1, _ := flate.NewWriter(&cw1, lvl) // Streamed compressor
	zw2, _ := flate.NewWriter(&cw2, lvl) // Chunked compressor
	buf := make([]byte, chnkSize)
	for {
		// Read a full chunks worth of data.
		cnt, err := io.ReadFull(r, buf)
		strmRec.rawSize += int64(cnt)
		if err == io.EOF {
			break
		}

		// Write chunk to both compressors.
		if _, err := zw1.Write(buf[:cnt]); err != nil {
			log.Fatal(err)
		}
		if _, err := zw2.Write(buf[:cnt]); err != nil {
			log.Fatal(err)
		}

		// Flush the chunked compressor, append the record, and reset.
		if err := zw2.Flush(); err != nil {
			log.Fatal(err)
		}
		chnkRecs = append(chnkRecs, indexRecord{rawSize: int64(cnt), compSize: int64(cw2)})
		cw2 = 0
		zw2.Reset(&cw2)

		if err == io.ErrUnexpectedEOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
	}

	// Flush the streamed compressor and record the compressed size.
	if err := zw1.Flush(); err != nil {
		log.Fatal(err)
	}
	strmRec.compSize = int64(cw1)
	return strmRec, chnkRecs
}

// compress compresses the input buffer at the given level.
func compress(b []byte, lvl int) []byte {
	var buf bytes.Buffer
	w, err := flate.NewWriter(&buf, lvl)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := io.Copy(w, bytes.NewReader(b)); err != nil {
		log.Fatal(err)
	}
	if err := w.Close(); err != nil {
		log.Fatal(err)
	}
	return buf.Bytes()
}

// encode encodes the input using XFLATE's meta encoding.
func encode(b []byte) []byte {
	var buf bytes.Buffer
	mw := meta.NewWriter(&buf)
	mw.FinalMode = meta.FinalMeta
	if _, err := io.Copy(mw, bytes.NewReader(b)); err != nil {
		log.Fatal(err)
	}
	if err := mw.Close(); err != nil {
		log.Fatal(err)
	}
	return buf.Bytes()
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"gopherpit.com/gopherpit/pkg/webutils"
	"gopherpit.com/gopherpit/services/user"
)

type auditRecord struct {
	Time     time.Time   `json:"time,omitempty"`
	UserID   string      `json:"user-id,omitempty"`
	Username string      `json:"username,omitempty"`
	Info     interface{} `json:"info,omitempty"`
	IPs      string      `json:"ips,omitempty"`
	Action   string      `json:"action,omitempty"`
	Message  string      `json:"massage,omitempty"`
}

func (s Server) audit(r *http.Request, info interface{}, action, message string) {
	if s.auditLogger == nil {
		return
	}
	var userID, email string
	u, _, err := s.user(r)
	if err != nil && err != user.UserNotFound {
		s.logger.Errorf("audit: get user: %s", err)
		return
	}
	if u != nil {
		userID = u.ID
		email = u.Email
	}
	record, err := json.Marshal(auditRecord{
		Time:     time.Now().UTC(),
		UserID:   userID,
		Username: email,
		Info:     info,
		IPs:      webutils.GetIPs(r),
		Action:   action,
		Message:  message,
	})
	if err != nil {
		s.logger.Errorf("audit: json encode: %s", err)
		return
	}
	s.auditLogger.Info(string(record))
}

func (s Server) auditf(r *http.Request, info interface{}, action, format string, a ...interface{}) {
	s.audit(r, info, action, fmt.Sprintf(format, a...))
}
<commit_msg>Log both username and e-mail to audit log<commit_after>// Copyright (c) 2016, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"gopherpit.com/gopherpit/pkg/webutils"
	"gopherpit.com/gopherpit/services/user"
)

type auditRecord struct {
	Time     time.Time   `json:"time,omitempty"`
	UserID   string      `json:"user-id,omitempty"`
	Username string      `json:"username,omitempty"`
	Email    string      `json:"email,omitempty"`
	Info     interface{} `json:"info,omitempty"`
	IPs      string      `json:"ips,omitempty"`
	Action   string      `json:"action,omitempty"`
	Message  string      `json:"massage,omitempty"`
}

func (s Server) audit(r *http.Request, info interface{}, action, message string) {
	if s.auditLogger == nil {
		return
	}
	var userID, username, email string
	u, _, err := s.user(r)
	if err != nil && err != user.UserNotFound {
		s.logger.Errorf("audit: get user: %s", err)
		return
	}
	if u != nil {
		userID = u.ID
		username = u.Username
		email = u.Email
	}
	record, err := json.Marshal(auditRecord{
		Time:     time.Now().UTC(),
		UserID:   userID,
		Username: username,
		Email:    email,
		Info:     info,
		IPs:      webutils.GetIPs(r),
		Action:   action,
		Message:  message,
	})
	if err != nil {
		s.logger.Errorf("audit: json encode: %s", err)
		return
	}
	s.auditLogger.Info(string(record))
}

func (s Server) auditf(r *http.Request, info interface{}, action, format string, a ...interface{}) {
	s.audit(r, info, action, fmt.Sprintf(format, a...))
}
<|endoftext|>


<commit_before>package bsw

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

type bingMessage struct {
	D bingResults `json:"D"`
}

type bingResults struct {
	Results []bingResult `json:"Results"`
}

type bingResult struct {
	Metadata    bingMetadata `json:"__Metadata"`
	ID          string       `json:"id"`
	Title       string       `json:"Title"`
	Description string       `json:"Description"`
	DisplayURL  string       `json:"DisplayUrl"`
	URL         string       `json:"Url"`
}

type bingMetadata struct {
	Uri  string `json:"Uri"`
	Type string `json:"Type"`
}

const azureURL = "https://api.datamarket.azure.com"

// FindBingSearchPath attempts an authenticated search request to two different Bing API paths. If and when a
// search is successfull, that path will be returned. If no path is valid this function
// returns an error.
func FindBingSearchPath(key string) (string, error) {
	paths := []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
	query := "?Query=%27I<3BSW%27"
	for _, path := range paths {
		fullURL := azureURL + path + query
		client := &http.Client{}
		req, err := http.NewRequest("GET", fullURL, nil)
		if err != nil {
			return "", err
		}
		req.SetBasicAuth(key, key)
		resp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == 200 {
			return path, nil
		}
	}
	return "", errors.New("invalid Bing API key")
}

// BingAPI uses the bing search API and 'ip' search operator to find alternate hostnames for
// a single IP.
func BingAPI(ip, key, path string) (string, Results, error) {
	task := "bing API"
	results := Results{}
	client := &http.Client{}
	req, err := http.NewRequest("GET", azureURL+path+"?Query=%27ip:"+ip+"%27&$top=50&Adult=%27off%27&$format=json", nil)
	if err != nil {
		return task, results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return task, results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return task, results, err
	}
	m := &bingMessage{}
	if err = json.Unmarshal(body, &m); err != nil {
		return task, results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.URL)
		if err == nil {
			results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
		}
	}
	return task, results, nil
}

func Bing(ip string) (string, Results, error) {
	task := "bing"
	results := Results{}
	resp, err := http.Get("http://www.bing.com/search?q=ip:" + ip)
	if err != nil {
		return task, results, err
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return task, results, err
	}
	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
		u, err := url.Parse(s.Text())
		if err == nil && u.Host != "" {
			results = append(results, Result{
				Source:   task,
				IP:       ip,
				Hostname: u.Host,
			})
		}
	})
	return task, results, err
}
<commit_msg>Fixes #8<commit_after>package bsw

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

type bingMessage struct {
	D bingResults `json:"D"`
}

type bingResults struct {
	Results []bingResult `json:"Results"`
}

type bingResult struct {
	Metadata    bingMetadata `json:"__Metadata"`
	ID          string       `json:"id"`
	Title       string       `json:"Title"`
	Description string       `json:"Description"`
	DisplayURL  string       `json:"DisplayUrl"`
	URL         string       `json:"Url"`
}

type bingMetadata struct {
	Uri  string `json:"Uri"`
	Type string `json:"Type"`
}

const azureURL = "https://api.datamarket.azure.com"

// FindBingSearchPath attempts an authenticated search request to two different Bing API paths. If and when a
// search is successfull, that path will be returned. If no path is valid this function
// returns an error.
func FindBingSearchPath(key string) (string, error) {
	paths := []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
	query := "?Query=%27I<3BSW%27"
	for _, path := range paths {
		fullURL := azureURL + path + query
		client := &http.Client{}
		req, err := http.NewRequest("GET", fullURL, nil)
		if err != nil {
			return "", err
		}
		req.SetBasicAuth(key, key)
		resp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == 200 {
			return path, nil
		}
	}
	return "", errors.New("invalid Bing API key")
}

// BingAPIIP uses the bing search API and 'ip' search operator to find alternate hostnames for
// a single IP.
func BingAPIIP(ip, key, path string) (string, Results, error) {
	task := "bing API"
	results := Results{}
	client := &http.Client{}
	req, err := http.NewRequest("GET", azureURL+path+"?Query=%27ip:"+ip+"%27&$top=50&Adult=%27off%27&$format=json", nil)
	if err != nil {
		return task, results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return task, results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return task, results, err
	}
	m := &bingMessage{}
	if err = json.Unmarshal(body, &m); err != nil {
		return task, results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.URL)
		if err == nil {
			results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
		}
	}
	return task, results, nil
}

// BingAPIDomain uses the bing search API and 'domain' search operator to find hostnames for
// a single domain.
func BingAPIDomain(domain, key, path, server string) (string, Results, error) {
	task := "bing API"
	results := Results{}
	client := &http.Client{}
	req, err := http.NewRequest("GET", azureURL+path+"?Query=%27domain:"+domain+"%27&$top=50&Adult=%27off%27&$format=json", nil)
	if err != nil {
		return task, results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return task, results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return task, results, err
	}
	m := &bingMessage{}
	if err = json.Unmarshal(body, &m); err != nil {
		return task, results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.URL)
		if err == nil && u.Host != "" {
			ip, err := LookupName(u.Host, server)
			if err != nil || ip == "" {
				ip, err = LookupCname(u.Host, server)
				if err != nil || ip == "" {
					continue
				}
			}
			results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
		}
	}
	return task, results, nil
}

// BingIP uses bing's 'ip:' search operator and scrapes the HTML to find hostnames for an ip.
func BingIP(ip string) (string, Results, error) {
	task := "bing"
	results := Results{}
	resp, err := http.Get("http://www.bing.com/search?q=ip:" + ip)
	if err != nil {
		return task, results, err
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return task, results, err
	}
	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
		u, err := url.Parse(s.Text())
		if err == nil && u.Host != "" {
			results = append(results, Result{
				Source:   task,
				IP:       ip,
				Hostname: u.Host,
			})
		}
	})
	return task, results, err
}

// BingDomain uses bing's 'domain:' search operator and scrapes the HTML to find ips and hostnames for a domain.
func BingDomain(domain, server string) (string, Results, error) {
	task := "bing"
	results := Results{}
	resp, err := http.Get("http://www.bing.com/search?q=domain:" + domain)
	if err != nil {
		return task, results, err
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return task, results, err
	}
	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
		u, err := url.Parse(s.Text())
		if err == nil && u.Host != "" {
			ip, err := LookupName(u.Host, server)
			if err != nil || ip == "" {
				ip, err = LookupCname(u.Host, server)
				if err != nil || ip == "" {
					return
				}
			}
			results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
		}
	})
	return task, results, err
}
<|endoftext|>


<commit_before>package libvirt

import (
	"log"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

// Provider libvirt
func Provider() terraform.ResourceProvider {
	return &schema.Provider{
		Schema: map[string]*schema.Schema{
			"uri": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("LIBVIRT_DEFAULT_URI", nil),
				Description: "libvirt connection URI for operations. See https://libvirtc.org/uri.html",
			},
		},

		ResourcesMap: map[string]*schema.Resource{
			"libvirt_domain":         resourceLibvirtDomain(),
			"libvirt_volume":         resourceLibvirtVolume(),
			"libvirt_network":        resourceLibvirtNetwork(),
			"libvirt_pool":           resourceLibvirtPool(),
			"libvirt_cloudinit_disk": resourceCloudInitDisk(),
			"libvirt_ignition":       resourceIgnition(),
		},

		DataSourcesMap: map[string]*schema.Resource{
			"libvirt_network_dns_host_template":        datasourceLibvirtNetworkDNSHostTemplate(),
			"libvirt_network_dns_srv_template":         datasourceLibvirtNetworkDNSSRVTemplate(),
			"libvirt_network_dnsmasq_options_template": datasourceLibvirtNetworkDnsmasqOptionsTemplate(),
		},

		ConfigureFunc: providerConfigure,
	}
}

// uri -> client for multi instance support
// (we share the same client for the same uri)
var globalClientMap = make(map[string]*Client)

// CleanupLibvirtConnections closes libvirt clients for all URIs
func CleanupLibvirtConnections() {
	for uri, client := range globalClientMap {
		log.Printf("[DEBUG] cleaning up connection for URI: %s", uri)
		alive, err := client.libvirtc.IsAlive()
		if err != nil {
			log.Printf("[ERROR] cannot determine libvirt connection status: %v", err)
		}
		if alive {
			ret, err := client.libvirtc.Close()
			if err != nil {
				log.Printf("[ERROR] cannot close libvirt connection %d - %v", ret, err)
			}
		}
	}
}

func providerConfigure(d *schema.ResourceData) (interface{}, error) {
	config := Config{
		URI: d.Get("uri").(string),
	}
	log.Printf("[DEBUG] Configuring provider for '%s': %v", config.URI, d)

	if client, ok := globalClientMap[config.URI]; ok {
		log.Printf("[DEBUG] Reusing client for uri: '%s'", config.URI)
		return client, nil
	}

	client, err := config.Client()
	if err != nil {
		return nil, err
	}
	globalClientMap[config.URI] = client

	return client, nil
}
<commit_msg>port - close our new second connection too<commit_after>package libvirt

import (
	"log"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

// Provider libvirt
func Provider() terraform.ResourceProvider {
	return &schema.Provider{
		Schema: map[string]*schema.Schema{
			"uri": {
				Type:        schema.TypeString,
				Required:    true,
				DefaultFunc: schema.EnvDefaultFunc("LIBVIRT_DEFAULT_URI", nil),
				Description: "libvirt connection URI for operations. See https://libvirtc.org/uri.html",
			},
		},

		ResourcesMap: map[string]*schema.Resource{
			"libvirt_domain":         resourceLibvirtDomain(),
			"libvirt_volume":         resourceLibvirtVolume(),
			"libvirt_network":        resourceLibvirtNetwork(),
			"libvirt_pool":           resourceLibvirtPool(),
			"libvirt_cloudinit_disk": resourceCloudInitDisk(),
			"libvirt_ignition":       resourceIgnition(),
		},

		DataSourcesMap: map[string]*schema.Resource{
			"libvirt_network_dns_host_template":        datasourceLibvirtNetworkDNSHostTemplate(),
			"libvirt_network_dns_srv_template":         datasourceLibvirtNetworkDNSSRVTemplate(),
			"libvirt_network_dnsmasq_options_template": datasourceLibvirtNetworkDnsmasqOptionsTemplate(),
		},

		ConfigureFunc: providerConfigure,
	}
}

// uri -> client for multi instance support
// (we share the same client for the same uri)
var globalClientMap = make(map[string]*Client)

// CleanupLibvirtConnections closes libvirt clients for all URIs
func CleanupLibvirtConnections() {
	for uri, client := range globalClientMap {
		log.Printf("[DEBUG] cleaning up connection for URI: %s", uri)
		alive, err := client.libvirtc.IsAlive()
		if err != nil {
			log.Printf("[ERROR] cannot determine libvirt connection status: %v", err)
		}
		if alive {
			ret, err := client.libvirtc.Close()
			if err != nil {
				log.Printf("[ERROR] cannot close libvirt connection %d - %v", ret, err)
			}
		}
		// TODO: Confirm appropriate IsAlive() validation
		err = client.libvirt.ConnectClose()
		if err != nil {
			log.Printf("[ERROR] cannot close libvirt connection: %v", err)
		}
	}
}

func providerConfigure(d *schema.ResourceData) (interface{}, error) {
	config := Config{
		URI: d.Get("uri").(string),
	}
	log.Printf("[DEBUG] Configuring provider for '%s': %v", config.URI, d)

	if client, ok := globalClientMap[config.URI]; ok {
		log.Printf("[DEBUG] Reusing client for uri: '%s'", config.URI)
		return client, nil
	}

	client, err := config.Client()
	if err != nil {
		return nil, err
	}
	globalClientMap[config.URI] = client

	return client, nil
}
<|endoftext|>