<commit_before>package venom

import (
	"sort"
	"strings"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

func allKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

//
// Better version of viper.AutomaticEnv that searches FOO_BAR for every --foo-bar key in
// addition to the default FOO-BAR.
//
// Note that it must be called *after* all flags are added.
//
func automaticEnv(flags *pflag.FlagSet, v *viper.Viper) {
	replaceMap := make(map[string]string, flags.NFlag())
	flags.VisitAll(func(f *pflag.Flag) {
		name := strings.ToUpper(f.Name)
		replaceMap[name] = sanitize(name)
	})

	keys := allKeys(replaceMap)

	// Reverse sort keys, this is to make sure foo-bar comes before foo. This is to prevent
	// foo being triggered when foo-bar is given to string replacer.
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	values := make([]string, 0, 2*len(keys))
	for _, k := range keys {
		values = append(values, k)
		values = append(values, replaceMap[k])
	}

	v.SetEnvKeyReplacer(strings.NewReplacer(values...))
	v.AutomaticEnv()
}
<commit_msg>Use single append<commit_after>package venom

import (
	"sort"
	"strings"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

func allKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

//
// Better version of viper.AutomaticEnv that searches FOO_BAR for every --foo-bar key in
// addition to the default FOO-BAR.
//
// Note that it must be called *after* all flags are added.
//
func automaticEnv(flags *pflag.FlagSet, v *viper.Viper) {
	replaceMap := make(map[string]string, flags.NFlag())
	flags.VisitAll(func(f *pflag.Flag) {
		name := strings.ToUpper(f.Name)
		replaceMap[name] = sanitize(name)
	})

	keys := allKeys(replaceMap)

	// Reverse sort keys, this is to make sure foo-bar comes before foo. This is to prevent
	// foo being triggered when foo-bar is given to string replacer.
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	values := make([]string, 0, 2*len(keys))
	for _, k := range keys {
		values = append(values, k, replaceMap[k])
	}

	v.SetEnvKeyReplacer(strings.NewReplacer(values...))
	v.AutomaticEnv()
}
<|endoftext|>


<commit_before>package fauxgl

type Voxel struct {
	X, Y, Z int
	Color   Color
}

type voxelAxis int

const (
	_ voxelAxis = iota
	voxelX
	voxelY
	voxelZ
)

type voxelNormal struct {
	Axis voxelAxis
	Sign int
}

var (
	voxelPosX = voxelNormal{voxelX, 1}
	voxelNegX = voxelNormal{voxelX, -1}
	voxelPosY = voxelNormal{voxelY, 1}
	voxelNegY = voxelNormal{voxelY, -1}
	voxelPosZ = voxelNormal{voxelZ, 1}
	voxelNegZ = voxelNormal{voxelZ, -1}
)

type voxelPlane struct {
	Normal   voxelNormal
	Position int
	Color    Color
}

type voxelFace struct {
	I0, J0 int
	I1, J1 int
}

func NewVoxelMesh(voxels []Voxel) *Mesh {
	type key struct {
		X, Y, Z int
	}

	// create lookup table
	lookup := make(map[key]bool)
	for _, v := range voxels {
		lookup[key{v.X, v.Y, v.Z}] = true
	}

	// find exposed faces
	planeFaces := make(map[voxelPlane][]voxelFace)
	for _, v := range voxels {
		if !lookup[key{v.X + 1, v.Y, v.Z}] {
			plane := voxelPlane{voxelPosX, v.X, v.Color}
			face := voxelFace{v.Y, v.Z, v.Y, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X - 1, v.Y, v.Z}] {
			plane := voxelPlane{voxelNegX, v.X, v.Color}
			face := voxelFace{v.Y, v.Z, v.Y, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y + 1, v.Z}] {
			plane := voxelPlane{voxelPosY, v.Y, v.Color}
			face := voxelFace{v.X, v.Z, v.X, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y - 1, v.Z}] {
			plane := voxelPlane{voxelNegY, v.Y, v.Color}
			face := voxelFace{v.X, v.Z, v.X, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y, v.Z + 1}] {
			plane := voxelPlane{voxelPosZ, v.Z, v.Color}
			face := voxelFace{v.X, v.Y, v.X, v.Y}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y, v.Z - 1}] {
			plane := voxelPlane{voxelNegZ, v.Z, v.Color}
			face := voxelFace{v.X, v.Y, v.X, v.Y}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
	}

	var triangles []*Triangle
	var lines []*Line

	// find large rectangles, triangulate and outline
	for plane, faces := range planeFaces {
		faces = combineVoxelFaces(faces)
		lines = append(lines, outlineVoxelFaces(plane, faces)...)
		triangles = append(triangles, triangulateVoxelFaces(plane, faces)...)
	}

	// remove t-junctions in mesh
	if false {
		vertexes := make(map[Vector]bool)
		for _, t := range triangles {
			vertexes[t.V1.Position] = true
			vertexes[t.V2.Position] = true
			vertexes[t.V3.Position] = true
		}
		for v := range vertexes {
			for _, t := range triangles {
				if pointOnSegment(v, t.V1.Position, t.V2.Position) {
					u := NewTriangle(t.V1, t.V2, t.V3)
					u.V1.Position = v
					t.V2.Position = v
					triangles = append(triangles, u)
				}
				if pointOnSegment(v, t.V2.Position, t.V3.Position) {
					u := NewTriangle(t.V1, t.V2, t.V3)
					u.V2.Position = v
					t.V3.Position = v
					triangles = append(triangles, u)
				}
				if pointOnSegment(v, t.V3.Position, t.V1.Position) {
					u := NewTriangle(t.V1, t.V2, t.V3)
					u.V3.Position = v
					t.V1.Position = v
					triangles = append(triangles, u)
				}
			}
		}
	}

	return NewMesh(triangles, lines)
}

func pointOnSegment(p, v0, v1 Vector) bool {
	// v0 to v1 must be an axis aligned segment
	if v0.X == v1.X && v0.X == p.X && v0.Y == v1.Y && v0.Y == p.Y {
		z0, z1 := v0.Z, v1.Z
		return (p.Z > z0 && p.Z < z1) || (p.Z < z0 && p.Z > z1)
	}
	if v0.X == v1.X && v0.X == p.X && v0.Z == v1.Z && v0.Z == p.Z {
		y0, y1 := v0.Y, v1.Y
		return (p.Y > y0 && p.Y < y1) || (p.Y < y0 && p.Y > y1)
	}
	if v0.Y == v1.Y && v0.Y == p.Y && v0.Z == v1.Z && v0.Z == p.Z {
		x0, x1 := v0.X, v1.X
		return (p.X > x0 && p.X < x1) || (p.X < x0 && p.X > x1)
	}
	return false
}

func combineVoxelFaces(faces []voxelFace) []voxelFace {
	// determine bounding box
	i0 := faces[0].I0
	j0 := faces[0].J0
	i1 := faces[0].I1
	j1 := faces[0].J1
	for _, f := range faces {
		if f.I0 < i0 {
			i0 = f.I0
		}
		if f.J0 < j0 {
			j0 = f.J0
		}
		if f.I1 > i1 {
			i1 = f.I1
		}
		if f.J1 > j1 {
			j1 = f.J1
		}
	}
	// create arrays
	nj := j1 - j0 + 1
	ni := i1 - i0 + 1
	a := make([][]int, nj)
	w := make([][]int, nj)
	h := make([][]int, nj)
	for j := range a {
		a[j] = make([]int, ni)
		w[j] = make([]int, ni)
		h[j] = make([]int, ni)
	}
	// populate array
	count := 0
	for _, f := range faces {
		for j := f.J0; j <= f.J1; j++ {
			for i := f.I0; i <= f.I1; i++ {
				a[j-j0][i-i0] = 1
				count++
			}
		}
	}
	// find rectangles
	var result []voxelFace
	for count > 0 {
		var maxArea int
		var maxFace voxelFace
		for j := 0; j < nj; j++ {
			for i := 0; i < ni; i++ {
				if a[j][i] == 0 {
					continue
				}
				if j == 0 {
					h[j][i] = 1
				} else {
					h[j][i] = h[j-1][i] + 1
				}
				if i == 0 {
					w[j][i] = 1
				} else {
					w[j][i] = w[j][i-1] + 1
				}
				minw := w[j][i]
				for dh := 0; dh < h[j][i]; dh++ {
					if w[j-dh][i] < minw {
						minw = w[j-dh][i]
					}
					area := (dh + 1) * minw
					if area > maxArea {
						maxArea = area
						maxFace = voxelFace{
							i0 + i - minw + 1, j0 + j - dh, i0 + i, j0 + j}
					}
				}
			}
		}
		result = append(result, maxFace)
		for j := maxFace.J0; j <= maxFace.J1; j++ {
			for i := maxFace.I0; i <= maxFace.I1; i++ {
				a[j-j0][i-i0] = 0
				count--
			}
		}
		for j := 0; j < nj; j++ {
			for i := 0; i < ni; i++ {
				w[j][i] = 0
				h[j][i] = 0
			}
		}
	}
	return result
}

func triangulateVoxelFaces(plane voxelPlane, faces []voxelFace) []*Triangle {
	triangles := make([]*Triangle, len(faces)*2)
	k := float64(plane.Position) + float64(plane.Normal.Sign)*0.5
	for i, face := range faces {
		i0 := float64(face.I0) - 0.5
		j0 := float64(face.J0) - 0.5
		i1 := float64(face.I1) + 0.5
		j1 := float64(face.J1) + 0.5
		var p1, p2, p3, p4 Vector
		switch plane.Normal.Axis {
		case voxelX:
			p1 = Vector{k, i0, j0}
			p2 = Vector{k, i1, j0}
			p3 = Vector{k, i1, j1}
			p4 = Vector{k, i0, j1}
		case voxelY:
			p1 = Vector{i0, k, j1}
			p2 = Vector{i1, k, j1}
			p3 = Vector{i1, k, j0}
			p4 = Vector{i0, k, j0}
		case voxelZ:
			p1 = Vector{i0, j0, k}
			p2 = Vector{i1, j0, k}
			p3 = Vector{i1, j1, k}
			p4 = Vector{i0, j1, k}
		}
		if plane.Normal.Sign < 0 {
			p1, p2, p3, p4 = p4, p3, p2, p1
		}
		t1 := NewTriangleForPoints(p1, p2, p3)
		t2 := NewTriangleForPoints(p1, p3, p4)
		t1.V1.Color = plane.Color
		t1.V2.Color = plane.Color
		t1.V3.Color = plane.Color
		t2.V1.Color = plane.Color
		t2.V2.Color = plane.Color
		t2.V3.Color = plane.Color
		triangles[i*2+0] = t1
		triangles[i*2+1] = t2
	}
	return triangles
}

func outlineVoxelFaces(plane voxelPlane, faces []voxelFace) []*Line {
	// determine bounding box
	i0 := faces[0].I0
	j0 := faces[0].J0
	i1 := faces[0].I1
	j1 := faces[0].J1
	for _, f := range faces {
		if f.I0 < i0 {
			i0 = f.I0
		}
		if f.J0 < j0 {
			j0 = f.J0
		}
		if f.I1 > i1 {
			i1 = f.I1
		}
		if f.J1 > j1 {
			j1 = f.J1
		}
	}
	// padding
	i0--
	j0--
	i1++
	j1++
	// create array
	nj := j1 - j0 + 1
	ni := i1 - i0 + 1
	a := make([][]bool, nj)
	for j := range a {
		a[j] = make([]bool, ni)
	}
	// populate array
	for _, f := range faces {
		for j := f.J0; j <= f.J1; j++ {
			for i := f.I0; i <= f.I1; i++ {
				a[j-j0][i-i0] = true
			}
		}
	}
	var lines []*Line
	for sign := -1; sign <= 1; sign += 2 {
		// find "horizontal" lines
		for j := 1; j < nj-1; j++ {
			start := -1
			for i := 0; i < ni; i++ {
				if a[j][i] && !a[j+sign][i] {
					if start < 0 {
						start = i
					}
				} else if start >= 0 {
					end := i - 1
					ai := float64(i0+start) - 0.5
					bi := float64(i0+end) + 0.5
					jj := float64(j0+j) + 0.5*float64(sign)
					line := createVoxelOutline(plane, ai, jj, bi, jj)
					lines = append(lines, line)
					start = -1
				}
			}

		}
		// find "vertical" lines
		for i := 1; i < ni-1; i++ {
			start := -1
			for j := 0; j < nj; j++ {
				if a[j][i] && !a[j][i+sign] {
					if start < 0 {
						start = j
					}
				} else if start >= 0 {
					end := j - 1
					aj := float64(j0+start) - 0.5
					bj := float64(j0+end) + 0.5
					ii := float64(i0+i) + 0.5*float64(sign)
					line := createVoxelOutline(plane, ii, aj, ii, bj)
					lines = append(lines, line)
					start = -1
				}
			}
		}
	}
	return lines
}

func createVoxelOutline(plane voxelPlane, i0, j0, i1, j1 float64) *Line {
	k := float64(plane.Position) + float64(plane.Normal.Sign)*0.5
	var p1, p2 Vector
	switch plane.Normal.Axis {
	case voxelX:
		p1 = Vector{k, i0, j0}
		p2 = Vector{k, i1, j1}
	case voxelY:
		p1 = Vector{i0, k, j0}
		p2 = Vector{i1, k, j1}
	case voxelZ:
		p1 = Vector{i0, j0, k}
		p2 = Vector{i1, j1, k}
	}
	return NewLineForPoints(p1, p2)
}
<commit_msg>remove code<commit_after>package fauxgl

type Voxel struct {
	X, Y, Z int
	Color   Color
}

type voxelAxis int

const (
	_ voxelAxis = iota
	voxelX
	voxelY
	voxelZ
)

type voxelNormal struct {
	Axis voxelAxis
	Sign int
}

var (
	voxelPosX = voxelNormal{voxelX, 1}
	voxelNegX = voxelNormal{voxelX, -1}
	voxelPosY = voxelNormal{voxelY, 1}
	voxelNegY = voxelNormal{voxelY, -1}
	voxelPosZ = voxelNormal{voxelZ, 1}
	voxelNegZ = voxelNormal{voxelZ, -1}
)

type voxelPlane struct {
	Normal   voxelNormal
	Position int
	Color    Color
}

type voxelFace struct {
	I0, J0 int
	I1, J1 int
}

func NewVoxelMesh(voxels []Voxel) *Mesh {
	type key struct {
		X, Y, Z int
	}

	// create lookup table
	lookup := make(map[key]bool)
	for _, v := range voxels {
		lookup[key{v.X, v.Y, v.Z}] = true
	}

	// find exposed faces
	planeFaces := make(map[voxelPlane][]voxelFace)
	for _, v := range voxels {
		if !lookup[key{v.X + 1, v.Y, v.Z}] {
			plane := voxelPlane{voxelPosX, v.X, v.Color}
			face := voxelFace{v.Y, v.Z, v.Y, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X - 1, v.Y, v.Z}] {
			plane := voxelPlane{voxelNegX, v.X, v.Color}
			face := voxelFace{v.Y, v.Z, v.Y, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y + 1, v.Z}] {
			plane := voxelPlane{voxelPosY, v.Y, v.Color}
			face := voxelFace{v.X, v.Z, v.X, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y - 1, v.Z}] {
			plane := voxelPlane{voxelNegY, v.Y, v.Color}
			face := voxelFace{v.X, v.Z, v.X, v.Z}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y, v.Z + 1}] {
			plane := voxelPlane{voxelPosZ, v.Z, v.Color}
			face := voxelFace{v.X, v.Y, v.X, v.Y}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
		if !lookup[key{v.X, v.Y, v.Z - 1}] {
			plane := voxelPlane{voxelNegZ, v.Z, v.Color}
			face := voxelFace{v.X, v.Y, v.X, v.Y}
			planeFaces[plane] = append(planeFaces[plane], face)
		}
	}

	var triangles []*Triangle
	var lines []*Line

	// find large rectangles, triangulate and outline
	for plane, faces := range planeFaces {
		faces = combineVoxelFaces(faces)
		lines = append(lines, outlineVoxelFaces(plane, faces)...)
		triangles = append(triangles, triangulateVoxelFaces(plane, faces)...)
	}

	return NewMesh(triangles, lines)
}

func combineVoxelFaces(faces []voxelFace) []voxelFace {
	// determine bounding box
	i0 := faces[0].I0
	j0 := faces[0].J0
	i1 := faces[0].I1
	j1 := faces[0].J1
	for _, f := range faces {
		if f.I0 < i0 {
			i0 = f.I0
		}
		if f.J0 < j0 {
			j0 = f.J0
		}
		if f.I1 > i1 {
			i1 = f.I1
		}
		if f.J1 > j1 {
			j1 = f.J1
		}
	}
	// create arrays
	nj := j1 - j0 + 1
	ni := i1 - i0 + 1
	a := make([][]int, nj)
	w := make([][]int, nj)
	h := make([][]int, nj)
	for j := range a {
		a[j] = make([]int, ni)
		w[j] = make([]int, ni)
		h[j] = make([]int, ni)
	}
	// populate array
	count := 0
	for _, f := range faces {
		for j := f.J0; j <= f.J1; j++ {
			for i := f.I0; i <= f.I1; i++ {
				a[j-j0][i-i0] = 1
				count++
			}
		}
	}
	// find rectangles
	var result []voxelFace
	for count > 0 {
		var maxArea int
		var maxFace voxelFace
		for j := 0; j < nj; j++ {
			for i := 0; i < ni; i++ {
				if a[j][i] == 0 {
					continue
				}
				if j == 0 {
					h[j][i] = 1
				} else {
					h[j][i] = h[j-1][i] + 1
				}
				if i == 0 {
					w[j][i] = 1
				} else {
					w[j][i] = w[j][i-1] + 1
				}
				minw := w[j][i]
				for dh := 0; dh < h[j][i]; dh++ {
					if w[j-dh][i] < minw {
						minw = w[j-dh][i]
					}
					area := (dh + 1) * minw
					if area > maxArea {
						maxArea = area
						maxFace = voxelFace{
							i0 + i - minw + 1, j0 + j - dh, i0 + i, j0 + j}
					}
				}
			}
		}
		result = append(result, maxFace)
		for j := maxFace.J0; j <= maxFace.J1; j++ {
			for i := maxFace.I0; i <= maxFace.I1; i++ {
				a[j-j0][i-i0] = 0
				count--
			}
		}
		for j := 0; j < nj; j++ {
			for i := 0; i < ni; i++ {
				w[j][i] = 0
				h[j][i] = 0
			}
		}
	}
	return result
}

func triangulateVoxelFaces(plane voxelPlane, faces []voxelFace) []*Triangle {
	triangles := make([]*Triangle, len(faces)*2)
	k := float64(plane.Position) + float64(plane.Normal.Sign)*0.5
	for i, face := range faces {
		i0 := float64(face.I0) - 0.5
		j0 := float64(face.J0) - 0.5
		i1 := float64(face.I1) + 0.5
		j1 := float64(face.J1) + 0.5
		var p1, p2, p3, p4 Vector
		switch plane.Normal.Axis {
		case voxelX:
			p1 = Vector{k, i0, j0}
			p2 = Vector{k, i1, j0}
			p3 = Vector{k, i1, j1}
			p4 = Vector{k, i0, j1}
		case voxelY:
			p1 = Vector{i0, k, j1}
			p2 = Vector{i1, k, j1}
			p3 = Vector{i1, k, j0}
			p4 = Vector{i0, k, j0}
		case voxelZ:
			p1 = Vector{i0, j0, k}
			p2 = Vector{i1, j0, k}
			p3 = Vector{i1, j1, k}
			p4 = Vector{i0, j1, k}
		}
		if plane.Normal.Sign < 0 {
			p1, p2, p3, p4 = p4, p3, p2, p1
		}
		t1 := NewTriangleForPoints(p1, p2, p3)
		t2 := NewTriangleForPoints(p1, p3, p4)
		t1.V1.Color = plane.Color
		t1.V2.Color = plane.Color
		t1.V3.Color = plane.Color
		t2.V1.Color = plane.Color
		t2.V2.Color = plane.Color
		t2.V3.Color = plane.Color
		triangles[i*2+0] = t1
		triangles[i*2+1] = t2
	}
	return triangles
}

func outlineVoxelFaces(plane voxelPlane, faces []voxelFace) []*Line {
	// determine bounding box
	i0 := faces[0].I0
	j0 := faces[0].J0
	i1 := faces[0].I1
	j1 := faces[0].J1
	for _, f := range faces {
		if f.I0 < i0 {
			i0 = f.I0
		}
		if f.J0 < j0 {
			j0 = f.J0
		}
		if f.I1 > i1 {
			i1 = f.I1
		}
		if f.J1 > j1 {
			j1 = f.J1
		}
	}
	// padding
	i0--
	j0--
	i1++
	j1++
	// create array
	nj := j1 - j0 + 1
	ni := i1 - i0 + 1
	a := make([][]bool, nj)
	for j := range a {
		a[j] = make([]bool, ni)
	}
	// populate array
	for _, f := range faces {
		for j := f.J0; j <= f.J1; j++ {
			for i := f.I0; i <= f.I1; i++ {
				a[j-j0][i-i0] = true
			}
		}
	}
	var lines []*Line
	for sign := -1; sign <= 1; sign += 2 {
		// find "horizontal" lines
		for j := 1; j < nj-1; j++ {
			start := -1
			for i := 0; i < ni; i++ {
				if a[j][i] && !a[j+sign][i] {
					if start < 0 {
						start = i
					}
				} else if start >= 0 {
					end := i - 1
					ai := float64(i0+start) - 0.5
					bi := float64(i0+end) + 0.5
					jj := float64(j0+j) + 0.5*float64(sign)
					line := createVoxelOutline(plane, ai, jj, bi, jj)
					lines = append(lines, line)
					start = -1
				}
			}

		}
		// find "vertical" lines
		for i := 1; i < ni-1; i++ {
			start := -1
			for j := 0; j < nj; j++ {
				if a[j][i] && !a[j][i+sign] {
					if start < 0 {
						start = j
					}
				} else if start >= 0 {
					end := j - 1
					aj := float64(j0+start) - 0.5
					bj := float64(j0+end) + 0.5
					ii := float64(i0+i) + 0.5*float64(sign)
					line := createVoxelOutline(plane, ii, aj, ii, bj)
					lines = append(lines, line)
					start = -1
				}
			}
		}
	}
	return lines
}

func createVoxelOutline(plane voxelPlane, i0, j0, i1, j1 float64) *Line {
	k := float64(plane.Position) + float64(plane.Normal.Sign)*0.5
	var p1, p2 Vector
	switch plane.Normal.Axis {
	case voxelX:
		p1 = Vector{k, i0, j0}
		p2 = Vector{k, i1, j1}
	case voxelY:
		p1 = Vector{i0, k, j0}
		p2 = Vector{i1, k, j1}
	case voxelZ:
		p1 = Vector{i0, j0, k}
		p2 = Vector{i1, j1, k}
	}
	return NewLineForPoints(p1, p2)
}
<|endoftext|>


<commit_before>package webgo

import (
	"encoding/json"
	"errors"
	"html/template"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
)

type App struct {
	router        Router
	definitions   Definitions
	templates     *template.Template
	staticDir     string
	modules       Modules
	workDir       string
	tmpDir        string
	maxBodyLength int64
}

const (
	CT_JSON      = "application/json"
	CT_FORM      = "application/x-www-form-urlencoded"
	CT_MULTIPART = "multipart/form-data"
)

var app App
var CFG config
var LOGGER *Logger

func init() {

	var err error

	// Init CFG
	CFG = make(config)

	err = CFG.Read()

	if err != nil {
		panic(err)
	}

	// Init LOGGER
	LOGGER = NewLogger()

	cp := consoleProvider{}
	ep := emailProvider{}

	LOGGER.RegisterProvider(cp)
	LOGGER.RegisterProvider(ep)

	LOGGER.AddLogProvider(PROVIDER_CONSOLE)
	LOGGER.AddErrorProvider(PROVIDER_CONSOLE, PROVIDER_EMAIL)
	LOGGER.AddFatalProvider(PROVIDER_CONSOLE, PROVIDER_EMAIL)
	LOGGER.AddDebugProvider(PROVIDER_CONSOLE)

	// Init App
	templates := template.New("template")
	filepath.Walk("templates", func(path string, info os.FileInfo, err error) error {
		if strings.HasSuffix(path, ".html") {
			templates.ParseFiles(path)
		}
		return nil
	})
	app = App{}
	app.router = Router{make(Routes)}
	app.definitions = Definitions{}
	app.templates = templates
	app.staticDir = "public"
	app.modules = Modules{}

	app.workDir, err = os.Getwd()
	app.tmpDir = app.workDir + "/tmp"

	if CFG["maxBodyLength"] == "" {
		panic("maxBodyLength is empty")
	}
	app.maxBodyLength, err = strconv.ParseInt(CFG["maxBodyLength"], 10, 64)
	if err != nil {
		os.Exit(1)
	}

	//TODO: Проверить папку tmp, создать если необходимо
}

func parseRequest(ctx *Context, limit int64) (errorCode int, err error) {
	var body []byte

	defer func() {
		r := recover()
		if r != nil {
			errorCode = 400
			err = errors.New("Bad Request")
		}
	}()
	ctx.Request.Body = http.MaxBytesReader(ctx.Response, ctx.Request.Body, limit)

	if ctx.Request.Method == "GET" {
		err = ctx.Request.ParseForm()
		if err != nil {
			errorCode = 400
			return
		}

		// Копируем данные
		for i := range ctx.Request.Form {
			ctx.Query[i] = ctx.Request.Form[i]
		}

		return
	}

	switch ctx.ContentType {
	case CT_JSON:
		body, err = ioutil.ReadAll(ctx.Request.Body)
		if err != nil {
			errorCode = 400
			return
		}

		var data interface{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			errorCode = 400
			return
		}
		ctx._Body = body
		ctx.Body = data.(map[string]interface{})

		return
	case CT_FORM:
		err = ctx.Request.ParseForm()
		if err != nil {
			errorCode = 400
			return
		}

	case CT_MULTIPART:
		err = ctx.Request.ParseMultipartForm(limit)
		if err != nil {
			//TODO: 400 or 413
			errorCode = 400
			return
		}

		for _, fheaders := range ctx.Request.MultipartForm.File {
			for _, hdr := range fheaders {
				var infile multipart.File
				if infile, err = hdr.Open(); nil != err {
					errorCode = 500
					return
				}

				var outfile *os.File
				if outfile, err = os.Create(app.tmpDir + "/" + hdr.Filename); nil != err {
					errorCode = 500
					return
				}
				// 32K buffer copy
				var written int64
				if written, err = io.Copy(outfile, infile); nil != err {
					errorCode = 500
					return
				}

				ctx.Files = append(ctx.Files, File{FileName: hdr.Filename, Size: int64(written)})
			}
		}

	default:
		err = errors.New("Bad Request")
		errorCode = 400
		return
	}

	for i := range ctx.Request.Form {
		ctx.Body[i] = ctx.Request.Form[i]
	}

	return
}

func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	var vc reflect.Value
	var Action reflect.Value
	var middlewareGroup string

	method := r.Method
	path := r.URL.Path

	// Отдаем статику если был запрошен файл
	ext := filepath.Ext(path)
	if ext != "" {
		http.ServeFile(w, r, app.staticDir+filepath.Clean(path))
		return
	}

	if len(path) > 1 && path[len(path)-1:] == "/" {
		http.Redirect(w, r, path[:len(path)-1], 301)
		return
	}

	route := a.router.Match(method, path)
	if route == nil {
		http.Error(w, "", 404)
		return
	}

	vc = reflect.New(route.ControllerType)
	Action = vc.MethodByName(route.Options.Action)
	middlewareGroup = route.Options.MiddlewareGroup

	var err error
	ctx := Context{Response: w, Request: r, Query: make(map[string]interface{}), Body: make(map[string]interface{}), Params: route.Params, Method: method}
	ctx.ContentType = ctx.Request.Header.Get("Content-Type")
	ctx.ContentType, _, err = mime.ParseMediaType(ctx.ContentType)

	if err != nil {
		http.Error(w, "", 400)
		return
	}

	if route.Options.ContentType != "" {
		if route.Options.ContentType != ctx.ContentType {
			http.Error(w, "", 400)
			return
		}
	}

	Controller, ok := vc.Interface().(ControllerInterface)
	if !ok {
		LOGGER.Error(errors.New("controller is not ControllerInterface"))
		http.Error(w, "", 500)
		return
	}

	// Парсим запрос
	code, err := parseRequest(&ctx, app.maxBodyLength)
	if err != nil {
		http.Error(w, "", code)
		return
	}

	// Инициализация контекста
	Controller.Init(&ctx)

	// Запуск предобработчика
	if !Controller.Prepare() {
		return
	}

	// Запуск цепочки middleware
	if middlewareGroup != "" {
		isNext := app.definitions.Run(middlewareGroup, &ctx)
		if !isNext {
			return
		}
	}

	// Запуск Экшена
	in := make([]reflect.Value, 0)
	Action.Call(in)

	if ctx.ContentType == "multipart/form-data" {
		err = ctx.Files.RemoveAll()
		if err != nil {
			LOGGER.Error(err)
		}

		err = ctx.Request.MultipartForm.RemoveAll()
		if err != nil {
			LOGGER.Error(err)
		}
	}

	// Обрабатываем ошибки
	if ctx.error != nil {
		LOGGER.Error(err)
		http.Error(w, "", 500)
		return
	}

	// Запуск постобработчика
	Controller.Finish()
}

func RegisterMiddleware(name string, plugins ...MiddlewareInterface) {
	for _, plugin := range plugins {
		app.definitions.Register(name, plugin)
	}
}
func RegisterModule(name string, module ModuleInterface) {
	app.modules.RegisterModule(name, module)
}

func Get(url string, opts RouteOptions) {
	app.router.addRoute("GET", url, &opts)
}
func Post(url string, opts RouteOptions) {
	app.router.addRoute("POST", url, &opts)
}
func Put(url string, opts RouteOptions) {
	app.router.addRoute("PUT", url, &opts)
}
func Delete(url string, opts RouteOptions) {
	app.router.addRoute("DELETE", url, &opts)
}
func Options(url string, opts RouteOptions) {
	app.router.addRoute("OPTIONS", url, &opts)
}

func GetModule(str string) ModuleInterface {
	return app.modules[str]
}

func Run() {
	if CFG["port"] == "" {
		LOGGER.Fatal("Unknow port")
	}
	err := http.ListenAndServe(":"+CFG["port"], &app)
	if err != nil {
		LOGGER.Fatal(err)
	}
}
<commit_msg>Update<commit_after>package webgo

import (
	"encoding/json"
	"errors"
	"html/template"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
)

type App struct {
	router        Router
	definitions   Definitions
	templates     *template.Template
	staticDir     string
	modules       Modules
	workDir       string
	tmpDir        string
	maxBodyLength int64
}

const (
	CT_JSON      = "application/json"
	CT_FORM      = "application/x-www-form-urlencoded"
	CT_MULTIPART = "multipart/form-data"
)

var app App
var CFG config
var LOGGER *Logger

func init() {

	var err error

	// Init CFG
	CFG = make(config)

	err = CFG.Read()

	if err != nil {
		panic(err)
	}

	// Init LOGGER
	LOGGER = NewLogger()

	cp := consoleProvider{}
	ep := emailProvider{}

	LOGGER.RegisterProvider(cp)
	LOGGER.RegisterProvider(ep)

	LOGGER.AddLogProvider(PROVIDER_CONSOLE)
	LOGGER.AddErrorProvider(PROVIDER_CONSOLE, PROVIDER_EMAIL)
	LOGGER.AddFatalProvider(PROVIDER_CONSOLE, PROVIDER_EMAIL)
	LOGGER.AddDebugProvider(PROVIDER_CONSOLE)

	// Init App
	templates := template.New("template")
	filepath.Walk("templates", func(path string, info os.FileInfo, err error) error {
		if strings.HasSuffix(path, ".html") {
			templates.ParseFiles(path)
		}
		return nil
	})
	app = App{}
	app.router = Router{make(Routes)}
	app.definitions = Definitions{}
	app.templates = templates
	app.staticDir = "public"
	app.modules = Modules{}

	app.workDir, err = os.Getwd()
	app.tmpDir = app.workDir + "/tmp"

	if CFG["maxBodyLength"] == "" {
		panic("maxBodyLength is empty")
	}
	app.maxBodyLength, err = strconv.ParseInt(CFG["maxBodyLength"], 10, 64)
	if err != nil {
		os.Exit(1)
	}

	//TODO: Проверить папку tmp, создать если необходимо
}

func parseRequest(ctx *Context, limit int64) (errorCode int, err error) {
	var body []byte

	defer func() {
		r := recover()
		if r != nil {
			errorCode = 400
			err = errors.New("Bad Request")
		}
	}()
	ctx.Request.Body = http.MaxBytesReader(ctx.Response, ctx.Request.Body, limit)

	if ctx.Request.Method == "GET" {
		err = ctx.Request.ParseForm()
		if err != nil {
			errorCode = 400
			return
		}

		// Копируем данные
		for i := range ctx.Request.Form {
			ctx.Query[i] = ctx.Request.Form[i]
		}

		return
	}

	switch ctx.ContentType {
	case CT_JSON:
		body, err = ioutil.ReadAll(ctx.Request.Body)
		if err != nil {
			errorCode = 400
			return
		}

		var data interface{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			errorCode = 400
			return
		}
		ctx._Body = body
		ctx.Body = data.(map[string]interface{})

		return
	case CT_FORM:
		err = ctx.Request.ParseForm()
		if err != nil {
			errorCode = 400
			return
		}

	case CT_MULTIPART:
		err = ctx.Request.ParseMultipartForm(limit)
		if err != nil {
			//TODO: 400 or 413
			errorCode = 400
			return
		}

		for _, fheaders := range ctx.Request.MultipartForm.File {
			for _, hdr := range fheaders {
				var infile multipart.File
				if infile, err = hdr.Open(); nil != err {
					errorCode = 500
					return
				}

				var outfile *os.File
				if outfile, err = os.Create(app.tmpDir + "/" + hdr.Filename); nil != err {
					errorCode = 500
					return
				}
				// 32K buffer copy
				var written int64
				if written, err = io.Copy(outfile, infile); nil != err {
					errorCode = 500
					return
				}

				ctx.Files = append(ctx.Files, File{FileName: hdr.Filename, Size: int64(written)})
			}
		}

	default:
		err = errors.New("Bad Request")
		errorCode = 400
		return
	}

	for i := range ctx.Request.Form {
		ctx.Body[i] = ctx.Request.Form[i]
	}

	return
}

func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	var vc reflect.Value
	var Action reflect.Value
	var middlewareGroup string

	method := r.Method
	path := r.URL.Path

	// Отдаем статику если был запрошен файл
	ext := filepath.Ext(path)
	if ext != "" {
		http.ServeFile(w, r, app.staticDir+filepath.Clean(path))
		return
	}

	if len(path) > 1 && path[len(path)-1:] == "/" {
		http.Redirect(w, r, path[:len(path)-1], 301)
		return
	}

	route := a.router.Match(method, path)
	if route == nil {
		http.Error(w, "", 404)
		return
	}

	vc = reflect.New(route.ControllerType)
	Action = vc.MethodByName(route.Options.Action)
	middlewareGroup = route.Options.MiddlewareGroup

	var err error
	ctx := Context{Response: w, Request: r, Query: make(map[string]interface{}), Body: make(map[string]interface{}), Params: route.Params, Method: method}
	ctx.ContentType = ctx.Request.Header.Get("Content-Type")
	ctx.ContentType, _, err = mime.ParseMediaType(ctx.ContentType)

	if err != nil && method != "GET" {
		http.Error(w, "", 400)
		return
	}

	if route.Options.ContentType != "" && (method == "POST" || method == "PUT") {
		if route.Options.ContentType != ctx.ContentType {
			http.Error(w, "", 400)
			return
		}
	}

	Controller, ok := vc.Interface().(ControllerInterface)
	if !ok {
		LOGGER.Error(errors.New("controller is not ControllerInterface"))
		http.Error(w, "", 500)
		return
	}

	// Парсим запрос
	code, err := parseRequest(&ctx, app.maxBodyLength)
	if err != nil {
		http.Error(w, "", code)
		return
	}

	// Инициализация контекста
	Controller.Init(&ctx)

	// Запуск предобработчика
	if !Controller.Prepare() {
		return
	}

	// Запуск цепочки middleware
	if middlewareGroup != "" {
		isNext := app.definitions.Run(middlewareGroup, &ctx)
		if !isNext {
			return
		}
	}

	// Запуск Экшена
	in := make([]reflect.Value, 0)
	Action.Call(in)

	if ctx.ContentType == "multipart/form-data" {
		err = ctx.Files.RemoveAll()
		if err != nil {
			LOGGER.Error(err)
		}

		err = ctx.Request.MultipartForm.RemoveAll()
		if err != nil {
			LOGGER.Error(err)
		}
	}

	// Обрабатываем ошибки
	if ctx.error != nil {
		LOGGER.Error(err)
		http.Error(w, "", 500)
		return
	}

	// Запуск постобработчика
	Controller.Finish()
}

func RegisterMiddleware(name string, plugins ...MiddlewareInterface) {
	for _, plugin := range plugins {
		app.definitions.Register(name, plugin)
	}
}
func RegisterModule(name string, module ModuleInterface) {
	app.modules.RegisterModule(name, module)
}

func Get(url string, opts RouteOptions) {
	app.router.addRoute("GET", url, &opts)
}
func Post(url string, opts RouteOptions) {
	app.router.addRoute("POST", url, &opts)
}
func Put(url string, opts RouteOptions) {
	app.router.addRoute("PUT", url, &opts)
}
func Delete(url string, opts RouteOptions) {
	app.router.addRoute("DELETE", url, &opts)
}
func Options(url string, opts RouteOptions) {
	app.router.addRoute("OPTIONS", url, &opts)
}

func GetModule(str string) ModuleInterface {
	return app.modules[str]
}

func Run() {
	if CFG["port"] == "" {
		LOGGER.Fatal("Unknow port")
	}
	err := http.ListenAndServe(":"+CFG["port"], &app)
	if err != nil {
		LOGGER.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package epub

import (
	"archive/zip"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

const (
	containerFilename     = "container.xml"
	containerFileTemplate = `<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="%s/%s" media-type="application/oebps-package+xml" />
  </rootfiles>
</container>
`
	contentFolderName = "EPUB"
	// Permissions for any new directories we create
	dirPermissions = 0755
	// Permissions for any new files we create
	filePermissions   = 0644
	metaInfFolderName = "META-INF"
	mimetypeContent   = "application/epub+zip"
	mimetypeFilename  = "mimetype"
	pkgFilename       = "package.opf"
	sectionFileFormat = "section%04d.xhtml"
	tempDirPrefix     = "go-epub"
	xhtmlFolderName   = "xhtml"
)

func (e *epub) Write(destFilePath string) error {
	tempDir, err := ioutil.TempDir("", tempDirPrefix)
	defer os.Remove(tempDir)
	if err != nil {
		log.Fatalf("os.Remove error: %s", err)
	}

	err = createEpubFolders(tempDir)
	if err != nil {
		return err
	}

	err = writeMimetype(tempDir)
	if err != nil {
		return err
	}

	err = writeContainerFile(tempDir)
	if err != nil {
		return err
	}

	err = e.pkg.write(tempDir)
	if err != nil {
		return err
	}

	err = e.toc.write(tempDir)
	if err != nil {
		return err
	}

	err = e.writeSections(tempDir)
	if err != nil {
		return err
	}

	err = e.writeEpub(tempDir, destFilePath)
	if err != nil {
		return err
	}

	// TODO

	output, err := xml.MarshalIndent(e.toc.navDoc.xml, "", "  ")
	output = append([]byte(xhtmlDoctype), output...)

	//	output, err := xml.MarshalIndent(e.toc.ncxXml, "", "  ")

	//	output, err := xml.MarshalIndent(e.pkg.xml, "", "  ")

	output = append([]byte(xml.Header), output...)
	fmt.Println(string(output))

	return nil
}

func createEpubFolders(tempDir string) error {
	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
			xhtmlFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			metaInfFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	return nil
}

func writeContainerFile(tempDir string) error {
	containerFilePath := filepath.Join(tempDir, metaInfFolderName, containerFilename)
	if err := ioutil.WriteFile(
		containerFilePath,
		[]byte(
			fmt.Sprintf(
				containerFileTemplate,
				contentFolderName,
				pkgFilename,
			),
		),
		filePermissions,
	); err != nil {
		return err
	}

	return nil
}

/*
    type Rootfile struct {
        FullPath string `xml:"full-path,attr"`
        MediaType string `xml:"media-type,attr"`
    }

    type Rootfiles struct {
        Rootfile []Rootfile `xml:"rootfile"`
    }

    type Container struct {
        XMLName xml.Name `xml:"urn:oasis:names:tc:opendocument:xmlns:container container"`
        Version string `xml:"version,attr"`
        Rootfiles Rootfiles `xml:"rootfiles"`
    }

    v := &Container{Version: containerVersion}

    v.Rootfiles.Rootfile = append(
        v.Rootfiles.Rootfile,
        Rootfile{
            FullPath: filepath.Join(oebpsFolderName, contentFilename),
            MediaType: mediaTypeOebpsPackage,
        },
    )

    output, err := xml.MarshalIndent(v, "", `   `)
    if err != nil {
        return err
    }
    // Add the xml header to the output
    containerContent := append([]byte(xml.Header), output...)
    // It's generally nice to have files end with a newline
    containerContent = append(containerContent, "\n"...)

    containerFilePath := filepath.Join(metaInfFolderPath, containerFilename)
    if err := ioutil.WriteFile(containerFilePath, containerContent, filePermissions); err != nil {
        return err
    }

    return nil
}

/*
func writeContentFile(tempDir string) error {
    oebpsFolderPath := filepath.Join(tempDir, oebpsFolderName)
    if err := os.Mkdir(oebpsFolderPath, dirPermissions); err != nil {
        return err
    }

    type Package struct {
        XMLName xml.Name `xml:"http://www.idpf.org/2007/opf package"`
        UniqueIdentifier string `xml:"unique-identifier,attr"`
        Version string `xml:"version,attr"`
    }

    type Rootfile struct {
        FullPath string `xml:"full-path,attr"`
        MediaType string `xml:"media-type,attr"`
    }

    type Rootfiles struct {
        Rootfile []Rootfile `xml:"rootfile"`
    }

    type Container struct {
        XMLName xml.Name `xml:"urn:oasis:names:tc:opendocument:xmlns:container container"`
        Version string `xml:"version,attr"`
        Rootfiles Rootfiles `xml:"rootfiles"`
    }

    v := &Container{Version: containerVersion}

    v.Rootfiles.Rootfile = append(
        v.Rootfiles.Rootfile,
        Rootfile{
            FullPath: filepath.Join(oebpsFolderName, contentFilename),
            MediaType: mediaTypeOebpsPackage,
        },
    )

    output, err := xml.MarshalIndent(v, "", `   `)
    if err != nil {
        return err
    }
    // Add the xml header to the output
    containerContent := append([]byte(xml.Header), output...)
    // It's generally nice to have files end with a newline
    containerContent = append(containerContent, "\n"...)

    containerFilePath := filepath.Join(metaInfFolderPath, containerFilename)
    if err := ioutil.WriteFile(containerFilePath, containerContent, filePermissions); err != nil {
        return err
    }

    return nil
}
*/
func writeMimetype(tempDir string) error {
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)

	if err := ioutil.WriteFile(mimetypeFilePath, []byte(mimetypeContent), filePermissions); err != nil {
		return err
	}

	return nil
}

func (e *epub) writeSections(tempDir string) error {
	for i, section := range e.sections {
		sectionFilename := fmt.Sprintf(sectionFileFormat, i+1)
		sectionFilePath := filepath.Join(tempDir, contentFolderName, sectionFilename)

		if err := section.write(sectionFilePath); err != nil {
			return err
		}
	}

	return nil
}

func (e *epub) writeEpub(tempDir string, destFilePath string) error {
	f, err := os.Create(destFilePath)
	if err != nil {
		log.Fatalf("os.Create error: %s", err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Fatalf("os.File.Close error: %s", err)
		}
	}()

	z := zip.NewWriter(f)
	defer func() {
		if err := z.Close(); err != nil {
			log.Fatalf("zip.Writer.Close error: %s", err)
		}
	}()

	skipMimetypeFile := false

	var addFileToZip = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if skipMimetypeFile == true {
			// Skip the mimetype file
			if path == filepath.Join(tempDir, mimetypeFilename) {
				return nil
			}
		}

		// Get the path of the file relative to the folder we're zipping
		relativePath, err := filepath.Rel(tempDir, path)
		if err != nil {
			log.Fatalf("filepath.Rel error: %s", err)
		}

		// Only include regular files, not directories
		if !info.Mode().IsRegular() {
			return nil
		}

		r, err := os.Open(path)
		if err != nil {
			log.Fatalf("os.Open error: %s", err)
		}
		defer func() {
			if err := r.Close(); err != nil {
				log.Fatalf("os.File.Close error: %s", err)
			}
		}()

		w, err := z.Create(relativePath)
		if err != nil {
			log.Fatalf("zip.Writer.Create error: %s", err)
		}

		_, err = io.Copy(w, r)
		if err != nil {
			log.Fatalf("io.Copy error: %s", err)
		}

		return nil
	}

	// Add the mimetype file first
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)
	mimetypeInfo, err := os.Lstat(mimetypeFilePath)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}
	addFileToZip(mimetypeFilePath, mimetypeInfo, nil)

	skipMimetypeFile = true

	err = filepath.Walk(tempDir, addFileToZip)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}

	return nil
}
<commit_msg>Forgot to put sections in xhtml subfolder<commit_after>package epub

import (
	"archive/zip"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

const (
	containerFilename     = "container.xml"
	containerFileTemplate = `<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="%s/%s" media-type="application/oebps-package+xml" />
  </rootfiles>
</container>
`
	contentFolderName = "EPUB"
	// Permissions for any new directories we create
	dirPermissions = 0755
	// Permissions for any new files we create
	filePermissions   = 0644
	metaInfFolderName = "META-INF"
	mimetypeContent   = "application/epub+zip"
	mimetypeFilename  = "mimetype"
	pkgFilename       = "package.opf"
	sectionFileFormat = "section%04d.xhtml"
	tempDirPrefix     = "go-epub"
	xhtmlFolderName   = "xhtml"
)

func (e *epub) Write(destFilePath string) error {
	tempDir, err := ioutil.TempDir("", tempDirPrefix)
	defer os.Remove(tempDir)
	if err != nil {
		log.Fatalf("os.Remove error: %s", err)
	}

	err = createEpubFolders(tempDir)
	if err != nil {
		return err
	}

	err = writeMimetype(tempDir)
	if err != nil {
		return err
	}

	err = writeContainerFile(tempDir)
	if err != nil {
		return err
	}

	err = e.pkg.write(tempDir)
	if err != nil {
		return err
	}

	err = e.toc.write(tempDir)
	if err != nil {
		return err
	}

	err = e.writeSections(tempDir)
	if err != nil {
		return err
	}

	err = e.writeEpub(tempDir, destFilePath)
	if err != nil {
		return err
	}

	// TODO

	output, err := xml.MarshalIndent(e.toc.navDoc.xml, "", "  ")
	output = append([]byte(xhtmlDoctype), output...)

	//	output, err := xml.MarshalIndent(e.toc.ncxXml, "", "  ")

	//	output, err := xml.MarshalIndent(e.pkg.xml, "", "  ")

	output = append([]byte(xml.Header), output...)
	fmt.Println(string(output))

	return nil
}

func createEpubFolders(tempDir string) error {
	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
			xhtmlFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			metaInfFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	return nil
}

func writeContainerFile(tempDir string) error {
	containerFilePath := filepath.Join(tempDir, metaInfFolderName, containerFilename)
	if err := ioutil.WriteFile(
		containerFilePath,
		[]byte(
			fmt.Sprintf(
				containerFileTemplate,
				contentFolderName,
				pkgFilename,
			),
		),
		filePermissions,
	); err != nil {
		return err
	}

	return nil
}

/*
    type Rootfile struct {
        FullPath string `xml:"full-path,attr"`
        MediaType string `xml:"media-type,attr"`
    }

    type Rootfiles struct {
        Rootfile []Rootfile `xml:"rootfile"`
    }

    type Container struct {
        XMLName xml.Name `xml:"urn:oasis:names:tc:opendocument:xmlns:container container"`
        Version string `xml:"version,attr"`
        Rootfiles Rootfiles `xml:"rootfiles"`
    }

    v := &Container{Version: containerVersion}

    v.Rootfiles.Rootfile = append(
        v.Rootfiles.Rootfile,
        Rootfile{
            FullPath: filepath.Join(oebpsFolderName, contentFilename),
            MediaType: mediaTypeOebpsPackage,
        },
    )

    output, err := xml.MarshalIndent(v, "", `   `)
    if err != nil {
        return err
    }
    // Add the xml header to the output
    containerContent := append([]byte(xml.Header), output...)
    // It's generally nice to have files end with a newline
    containerContent = append(containerContent, "\n"...)

    containerFilePath := filepath.Join(metaInfFolderPath, containerFilename)
    if err := ioutil.WriteFile(containerFilePath, containerContent, filePermissions); err != nil {
        return err
    }

    return nil
}

/*
func writeContentFile(tempDir string) error {
    oebpsFolderPath := filepath.Join(tempDir, oebpsFolderName)
    if err := os.Mkdir(oebpsFolderPath, dirPermissions); err != nil {
        return err
    }

    type Package struct {
        XMLName xml.Name `xml:"http://www.idpf.org/2007/opf package"`
        UniqueIdentifier string `xml:"unique-identifier,attr"`
        Version string `xml:"version,attr"`
    }

    type Rootfile struct {
        FullPath string `xml:"full-path,attr"`
        MediaType string `xml:"media-type,attr"`
    }

    type Rootfiles struct {
        Rootfile []Rootfile `xml:"rootfile"`
    }

    type Container struct {
        XMLName xml.Name `xml:"urn:oasis:names:tc:opendocument:xmlns:container container"`
        Version string `xml:"version,attr"`
        Rootfiles Rootfiles `xml:"rootfiles"`
    }

    v := &Container{Version: containerVersion}

    v.Rootfiles.Rootfile = append(
        v.Rootfiles.Rootfile,
        Rootfile{
            FullPath: filepath.Join(oebpsFolderName, contentFilename),
            MediaType: mediaTypeOebpsPackage,
        },
    )

    output, err := xml.MarshalIndent(v, "", `   `)
    if err != nil {
        return err
    }
    // Add the xml header to the output
    containerContent := append([]byte(xml.Header), output...)
    // It's generally nice to have files end with a newline
    containerContent = append(containerContent, "\n"...)

    containerFilePath := filepath.Join(metaInfFolderPath, containerFilename)
    if err := ioutil.WriteFile(containerFilePath, containerContent, filePermissions); err != nil {
        return err
    }

    return nil
}
*/
func writeMimetype(tempDir string) error {
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)

	if err := ioutil.WriteFile(mimetypeFilePath, []byte(mimetypeContent), filePermissions); err != nil {
		return err
	}

	return nil
}

func (e *epub) writeSections(tempDir string) error {
	for i, section := range e.sections {
		sectionFilename := fmt.Sprintf(sectionFileFormat, i+1)
		sectionFilePath := filepath.Join(tempDir, contentFolderName, xhtmlFolderName, sectionFilename)

		if err := section.write(sectionFilePath); err != nil {
			return err
		}
	}

	return nil
}

func (e *epub) writeEpub(tempDir string, destFilePath string) error {
	f, err := os.Create(destFilePath)
	if err != nil {
		log.Fatalf("os.Create error: %s", err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Fatalf("os.File.Close error: %s", err)
		}
	}()

	z := zip.NewWriter(f)
	defer func() {
		if err := z.Close(); err != nil {
			log.Fatalf("zip.Writer.Close error: %s", err)
		}
	}()

	skipMimetypeFile := false

	var addFileToZip = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if skipMimetypeFile == true {
			// Skip the mimetype file
			if path == filepath.Join(tempDir, mimetypeFilename) {
				return nil
			}
		}

		// Get the path of the file relative to the folder we're zipping
		relativePath, err := filepath.Rel(tempDir, path)
		if err != nil {
			log.Fatalf("filepath.Rel error: %s", err)
		}

		// Only include regular files, not directories
		if !info.Mode().IsRegular() {
			return nil
		}

		r, err := os.Open(path)
		if err != nil {
			log.Fatalf("os.Open error: %s", err)
		}
		defer func() {
			if err := r.Close(); err != nil {
				log.Fatalf("os.File.Close error: %s", err)
			}
		}()

		w, err := z.Create(relativePath)
		if err != nil {
			log.Fatalf("zip.Writer.Create error: %s", err)
		}

		_, err = io.Copy(w, r)
		if err != nil {
			log.Fatalf("io.Copy error: %s", err)
		}

		return nil
	}

	// Add the mimetype file first
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)
	mimetypeInfo, err := os.Lstat(mimetypeFilePath)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}
	addFileToZip(mimetypeFilePath, mimetypeInfo, nil)

	skipMimetypeFile = true

	err = filepath.Walk(tempDir, addFileToZip)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package websocket

import (
	"github.com/Sirupsen/logrus"
	"github.com/gorilla/websocket"
	"github.com/pkg/errors"
	"github.com/verath/archipelago/lib/common"
	"net/http"
	"sync"
	"time"
)

// wsVersion is the version of the websocket protocol that the server
// is currently implementing. Used to make sure that the client "talks"
// the same version as we do.
const wsVersion = "2"

// The websocket.Upgrader used for all upgrades from http -> ws.
var wsUpgrader = websocket.Upgrader{
	HandshakeTimeout: 10 * time.Second,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

// WSConnectionHandler is a handler that handles new WSConnections.
type WSConnectionHandler interface {
	HandleWSConnection(*WSConnection) error
}

// WSConnectionHandlerFunc is a function that implements the WSConnectionHandler
// interface
type WSConnectionHandlerFunc func(*WSConnection) error

// HandleWSConnection calls the function itself.
func (f WSConnectionHandlerFunc) HandleWSConnection(conn *WSConnection) error {
	return f(conn)
}

// The UpgradeHandler is an http.Handler that attempts to upgrade requests
// to websocket connections. Successful upgrades are forwarded to the registered
// WSConnectionHandler.
type UpgradeHandler struct {
	logEntry *logrus.Entry

	connHandlerMu sync.RWMutex
	connHandler   WSConnectionHandler
}

// NewUpgradeHandler creates a new UpgradeHandler.
func NewUpgradeHandler(log *logrus.Logger) (*UpgradeHandler, error) {
	return &UpgradeHandler{
		logEntry: common.ModuleLogEntry(log, "websocket/UpgradeHandler"),
	}, nil
}

// SetConnectionHandler sets the handler for new connections.
func (h *UpgradeHandler) SetConnectionHandler(connHandler WSConnectionHandler) {
	h.connHandlerMu.Lock()
	defer h.connHandlerMu.Unlock()
	h.connHandler = connHandler
}

func (h *UpgradeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	clientWSVersion := r.URL.Query().Get("v")
	if clientWSVersion != wsVersion {
		h.logEntry.Warnf("Not upgrading to websocket, version "+
			"missmatch (client: %v, server: %v)", clientWSVersion, wsVersion)
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}
	wsConn, err := wsUpgrader.Upgrade(w, r, nil)
	if err != nil {
		h.logEntry.Warnf("Failed upgrading to websocket: %+v", err)
		return
	}
	if err := h.handleWSConn(wsConn); err != nil {
		h.logEntry.Errorf("Error handling ws connection: %+v", err)
	}
}

// handleWSConn wraps the gorilla websocket in our own WSConnection
// and forwards the connection to the connection handler.
func (h *UpgradeHandler) handleWSConn(wsConn *websocket.Conn) error {
	h.connHandlerMu.RLock()
	defer h.connHandlerMu.RUnlock()
	if h.connHandler == nil {
		return errors.New("connHandler was nil")
	}
	conn, err := newWSConnection(wsConn)
	if err != nil {
		return errors.Wrap(err, "Failed creating WSConnection")
	}
	return h.connHandler.HandleWSConnection(conn)
}
<commit_msg>Revert "Allow all origins to connect to ws endpoint."<commit_after>package websocket

import (
	"github.com/Sirupsen/logrus"
	"github.com/gorilla/websocket"
	"github.com/pkg/errors"
	"github.com/verath/archipelago/lib/common"
	"net/http"
	"sync"
)

// wsVersion is the version of the websocket protocol that the server
// is currently implementing. Used to make sure that the client "talks"
// the same version as we do.
const wsVersion = "2"

// The websocket.Upgrader used for all upgrades from http -> ws.
var wsUpgrader = websocket.Upgrader{}

// WSConnectionHandler is a handler that handles new WSConnections.
type WSConnectionHandler interface {
	HandleWSConnection(*WSConnection) error
}

// WSConnectionHandlerFunc is a function that implements the WSConnectionHandler
// interface
type WSConnectionHandlerFunc func(*WSConnection) error

// HandleWSConnection calls the function itself.
func (f WSConnectionHandlerFunc) HandleWSConnection(conn *WSConnection) error {
	return f(conn)
}

// The UpgradeHandler is an http.Handler that attempts to upgrade requests
// to websocket connections. Successful upgrades are forwarded to the registered
// WSConnectionHandler.
type UpgradeHandler struct {
	logEntry *logrus.Entry

	connHandlerMu sync.RWMutex
	connHandler   WSConnectionHandler
}

// NewUpgradeHandler creates a new UpgradeHandler.
func NewUpgradeHandler(log *logrus.Logger) (*UpgradeHandler, error) {
	return &UpgradeHandler{
		logEntry: common.ModuleLogEntry(log, "websocket/UpgradeHandler"),
	}, nil
}

// SetConnectionHandler sets the handler for new connections.
func (h *UpgradeHandler) SetConnectionHandler(connHandler WSConnectionHandler) {
	h.connHandlerMu.Lock()
	defer h.connHandlerMu.Unlock()
	h.connHandler = connHandler
}

func (h *UpgradeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	clientWSVersion := r.URL.Query().Get("v")
	if clientWSVersion != wsVersion {
		h.logEntry.Warnf("Not upgrading to websocket, version "+
			"missmatch (client: %v, server: %v)", clientWSVersion, wsVersion)
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}
	wsConn, err := wsUpgrader.Upgrade(w, r, nil)
	if err != nil {
		h.logEntry.Warnf("Failed upgrading to websocket: %+v", err)
		return
	}
	if err := h.handleWSConn(wsConn); err != nil {
		h.logEntry.Errorf("Error handling ws connection: %+v", err)
	}
}

// handleWSConn wraps the gorilla websocket in our own WSConnection
// and forwards the connection to the connection handler.
func (h *UpgradeHandler) handleWSConn(wsConn *websocket.Conn) error {
	h.connHandlerMu.RLock()
	defer h.connHandlerMu.RUnlock()
	if h.connHandler == nil {
		return errors.New("connHandler was nil")
	}
	conn, err := newWSConnection(wsConn)
	if err != nil {
		return errors.Wrap(err, "Failed creating WSConnection")
	}
	return h.connHandler.HandleWSConnection(conn)
}
<|endoftext|>


<commit_before>// Package bitseq provides a structure and utilities for representing long bitmask
// as sequence of run-lenght encoded blocks. It operates direclty on the encoded
// representation, it does not decode/encode.
package bitseq

import (
	"fmt"

	"github.com/docker/libnetwork/netutils"
)

// Block Sequence constants
// If needed we can think of making these configurable
const (
	blockLen      = 32
	blockBytes    = blockLen / 8
	blockMAX      = 1<<blockLen - 1
	blockFirstBit = 1 << (blockLen - 1)
)

// Handle contains the sequece representing the bitmask and its identifier
type Handle struct {
	ID   string
	Head *Sequence
}

// NewHandle returns an instance of the bitmask handler
func NewHandle(id string, numElements uint32) *Handle {
	return &Handle{
		ID: id,
		Head: &Sequence{
			Block: 0x0,
			Count: getNumBlocks(numElements),
			Next:  nil,
		},
	}
}

// Sequence reresents a recurring sequence of 32 bits long bitmasks
type Sequence struct {
	Block uint32    // block representing 4 byte long allocation bitmask
	Count uint32    // number of consecutive blocks
	Next  *Sequence // next sequence
}

// NewSequence returns a sequence initialized to represent a bitmaks of numElements bits
func NewSequence(numElements uint32) *Sequence {
	return &Sequence{Block: 0x0, Count: getNumBlocks(numElements), Next: nil}
}

// String returns a string representation of the block sequence starting from this block
func (s *Sequence) String() string {
	var nextBlock string
	if s.Next == nil {
		nextBlock = "end"
	} else {
		nextBlock = s.Next.String()
	}
	return fmt.Sprintf("(0x%x, %d)->%s", s.Block, s.Count, nextBlock)
}

// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
func (s *Sequence) GetAvailableBit() (bytePos, bitPos int) {
	if s.Block == blockMAX || s.Count == 0 {
		return -1, -1
	}
	bits := 0
	bitSel := uint32(blockFirstBit)
	for bitSel > 0 && s.Block&bitSel != 0 {
		bitSel >>= 1
		bits++
	}
	return bits / 8, bits % 8
}

// Equal checks if this sequence is equal to the passed one
func (s *Sequence) Equal(o *Sequence) bool {
	this := s
	other := o
	for this != nil {
		if other == nil {
			return false
		}
		if this.Block != other.Block || this.Count != other.Count {
			return false
		}
		this = this.Next
		other = other.Next
	}
	// Check if other is longer than this
	if other != nil {
		return false
	}
	return true
}

// ToByteArray converts the sequence into a byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) ToByteArray() ([]byte, error) {
	var bb []byte

	p := s
	for p != nil {
		bb = append(bb, netutils.U32ToA(p.Block)...)
		bb = append(bb, netutils.U32ToA(p.Count)...)
		p = p.Next
	}

	return bb, nil
}

// FromByteArray construct the sequence from the byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) FromByteArray(data []byte) error {
	l := len(data)
	if l%8 != 0 {
		return fmt.Errorf("cannot deserialize byte sequence of lenght %d", l)
	}

	p := s
	i := 0
	for {
		p.Block = netutils.ATo32(data[i : i+4])
		p.Count = netutils.ATo32(data[i+4 : i+8])
		i += 8
		if i == l {
			break
		}
		p.Next = &Sequence{}
		p = p.Next
	}

	return nil
}

// GetFirstAvailable returns the byte and bit position of the first unset bit
func (h *Handle) GetFirstAvailable() (int, int, error) {
	return GetFirstAvailable(h.Head)
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func (h *Handle) CheckIfAvailable(ordinal int) (int, int, error) {
	return CheckIfAvailable(h.Head, ordinal)
}

// PushReservation pushes the bit reservation inside the bitmask.
func (h *Handle) PushReservation(bytePos, bitPos int, release bool) {
	h.Head = PushReservation(bytePos, bitPos, h.Head, release)
}

// GetFirstAvailable looks for the first unset bit in passed mask
func GetFirstAvailable(head *Sequence) (int, int, error) {
	byteIndex := 0
	current := head
	for current != nil {
		if current.Block != blockMAX {
			bytePos, bitPos := current.GetAvailableBit()
			return byteIndex + bytePos, bitPos, nil
		}
		byteIndex += int(current.Count * blockBytes)
		current = current.Next
	}
	return -1, -1, fmt.Errorf("no bit available")
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func CheckIfAvailable(head *Sequence, ordinal int) (int, int, error) {
	bytePos := ordinal / 8
	bitPos := ordinal % 8

	// Find the Sequence containing this byte
	current, _, _, inBlockBytePos := findSequence(head, bytePos)

	if current != nil {
		// Check whether the bit corresponding to the ordinal address is unset
		bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
		if current.Block&bitSel == 0 {
			return bytePos, bitPos, nil
		}
	}

	return -1, -1, fmt.Errorf("requested bit is not available")
}

// Given the byte position and the sequences list head, return the pointer to the
// sequence containing the byte (current), the pointer to the previous sequence,
// the number of blocks preceding the block containing the byte inside the current sequence.
// If bytePos is outside of the list, function will return (nil, nil, 0, -1)
func findSequence(head *Sequence, bytePos int) (*Sequence, *Sequence, uint32, int) {
	// Find the Sequence containing this byte
	previous := head
	current := head
	n := bytePos
	for current.Next != nil && n >= int(current.Count*blockBytes) { // Nil check for less than 32 addresses masks
		n -= int(current.Count * blockBytes)
		previous = current
		current = current.Next
	}

	// If byte is outside of the list, let caller know
	if n >= int(current.Count*blockBytes) {
		return nil, nil, 0, -1
	}

	// Find the byte position inside the block and the number of blocks
	// preceding the block containing the byte inside this sequence
	precBlocks := uint32(n / blockBytes)
	inBlockBytePos := bytePos % blockBytes

	return current, previous, precBlocks, inBlockBytePos
}

// PushReservation pushes the bit reservation inside the bitmask.
// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
// Create a new block with the modified bit according to the operation (allocate/release).
// Create a new Sequence containing the new Block and insert it in the proper position.
// Remove current sequence if empty.
// Check if new Sequence can be merged with neighbour (previous/Next) sequences.
//
//
// Identify "current" Sequence containing block:
//                                      [prev seq] [current seq] [Next seq]
//
// Based on block position, resulting list of sequences can be any of three forms:
//
//        Block position                        Resulting list of sequences
// A) Block is first in current:         [prev seq] [new] [modified current seq] [Next seq]
// B) Block is last in current:          [prev seq] [modified current seq] [new] [Next seq]
// C) Block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [Next seq]
func PushReservation(bytePos, bitPos int, head *Sequence, release bool) *Sequence {
	// Store list's head
	newHead := head

	// Find the Sequence containing this byte
	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
	if current == nil {
		return newHead
	}

	// Construct updated block
	bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
	newBlock := current.Block
	if release {
		newBlock &^= bitSel
	} else {
		newBlock |= bitSel
	}

	// Quit if it was a redundant request
	if current.Block == newBlock {
		return newHead
	}

	// Current Sequence inevitably looses one block, upadate Count
	current.Count--

	// Create new sequence
	newSequence := &Sequence{Block: newBlock, Count: 1}

	// Insert the new sequence in the list based on block position
	if precBlocks == 0 { // First in sequence (A)
		newSequence.Next = current
		if current == head {
			newHead = newSequence
			previous = newHead
		} else {
			previous.Next = newSequence
		}
		removeCurrentIfEmpty(&newHead, newSequence, current)
		mergeSequences(previous)
	} else if precBlocks == current.Count-2 { // Last in sequence (B)
		newSequence.Next = current.Next
		current.Next = newSequence
		mergeSequences(current)
	} else { // In between the sequence (C)
		currPre := &Sequence{Block: current.Block, Count: precBlocks, Next: newSequence}
		currPost := current
		currPost.Count -= precBlocks
		newSequence.Next = currPost
		if currPost == head {
			newHead = currPre
		} else {
			previous.Next = currPre
		}
		// No merging or empty current possible here
	}

	return newHead
}

// Removes the current sequence from the list if empty, adjusting the head pointer if needed
func removeCurrentIfEmpty(head **Sequence, previous, current *Sequence) {
	if current.Count == 0 {
		if current == *head {
			*head = current.Next
		} else {
			previous.Next = current.Next
			current = current.Next
		}
	}
}

// Given a pointer to a Sequence, it checks if it can be merged with any following sequences
// It stops when no more merging is possible.
// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
func mergeSequences(seq *Sequence) {
	if seq != nil {
		// Merge all what possible from seq
		for seq.Next != nil && seq.Block == seq.Next.Block {
			seq.Count += seq.Next.Count
			seq.Next = seq.Next.Next
		}
		// Move to Next
		mergeSequences(seq.Next)
	}
}

// Serialize converts the sequence into a byte array
func Serialize(head *Sequence) ([]byte, error) {
	return nil, nil
}

// Deserialize decodes the byte array into a sequence
func Deserialize(data []byte) (*Sequence, error) {
	return nil, nil
}

func getNumBlocks(numBits uint32) uint32 {
	numBlocks := numBits / blockLen
	if numBits%blockLen != 0 {
		numBlocks++
	}
	return numBlocks
}
<commit_msg>Make bitseq.Handle thread-safe<commit_after>// Package bitseq provides a structure and utilities for representing long bitmask
// as sequence of run-lenght encoded blocks. It operates direclty on the encoded
// representation, it does not decode/encode.
package bitseq

import (
	"fmt"
	"sync"

	"github.com/docker/libnetwork/netutils"
)

// Block Sequence constants
// If needed we can think of making these configurable
const (
	blockLen      = 32
	blockBytes    = blockLen / 8
	blockMAX      = 1<<blockLen - 1
	blockFirstBit = 1 << (blockLen - 1)
)

// Handle contains the sequece representing the bitmask and its identifier
type Handle struct {
	ID   string
	Head *Sequence
	sync.Mutex
}

// NewHandle returns a thread-safe instance of the bitmask handler
func NewHandle(id string, numElements uint32) *Handle {
	return &Handle{
		ID: id,
		Head: &Sequence{
			Block: 0x0,
			Count: getNumBlocks(numElements),
			Next:  nil,
		},
	}
}

// Sequence reresents a recurring sequence of 32 bits long bitmasks
type Sequence struct {
	Block uint32    // block representing 4 byte long allocation bitmask
	Count uint32    // number of consecutive blocks
	Next  *Sequence // next sequence
}

// NewSequence returns a sequence initialized to represent a bitmaks of numElements bits
func NewSequence(numElements uint32) *Sequence {
	return &Sequence{Block: 0x0, Count: getNumBlocks(numElements), Next: nil}
}

// String returns a string representation of the block sequence starting from this block
func (s *Sequence) String() string {
	var nextBlock string
	if s.Next == nil {
		nextBlock = "end"
	} else {
		nextBlock = s.Next.String()
	}
	return fmt.Sprintf("(0x%x, %d)->%s", s.Block, s.Count, nextBlock)
}

// GetAvailableBit returns the position of the first unset bit in the bitmask represented by this sequence
func (s *Sequence) GetAvailableBit() (bytePos, bitPos int) {
	if s.Block == blockMAX || s.Count == 0 {
		return -1, -1
	}
	bits := 0
	bitSel := uint32(blockFirstBit)
	for bitSel > 0 && s.Block&bitSel != 0 {
		bitSel >>= 1
		bits++
	}
	return bits / 8, bits % 8
}

// Equal checks if this sequence is equal to the passed one
func (s *Sequence) Equal(o *Sequence) bool {
	this := s
	other := o
	for this != nil {
		if other == nil {
			return false
		}
		if this.Block != other.Block || this.Count != other.Count {
			return false
		}
		this = this.Next
		other = other.Next
	}
	// Check if other is longer than this
	if other != nil {
		return false
	}
	return true
}

// ToByteArray converts the sequence into a byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) ToByteArray() ([]byte, error) {
	var bb []byte

	p := s
	for p != nil {
		bb = append(bb, netutils.U32ToA(p.Block)...)
		bb = append(bb, netutils.U32ToA(p.Count)...)
		p = p.Next
	}

	return bb, nil
}

// FromByteArray construct the sequence from the byte array
// TODO (aboch): manage network/host order stuff
func (s *Sequence) FromByteArray(data []byte) error {
	l := len(data)
	if l%8 != 0 {
		return fmt.Errorf("cannot deserialize byte sequence of lenght %d", l)
	}

	p := s
	i := 0
	for {
		p.Block = netutils.ATo32(data[i : i+4])
		p.Count = netutils.ATo32(data[i+4 : i+8])
		i += 8
		if i == l {
			break
		}
		p.Next = &Sequence{}
		p = p.Next
	}

	return nil
}

// GetFirstAvailable returns the byte and bit position of the first unset bit
func (h *Handle) GetFirstAvailable() (int, int, error) {
	h.Lock()
	defer h.Unlock()
	return GetFirstAvailable(h.Head)
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func (h *Handle) CheckIfAvailable(ordinal int) (int, int, error) {
	h.Lock()
	defer h.Unlock()
	return CheckIfAvailable(h.Head, ordinal)
}

// PushReservation pushes the bit reservation inside the bitmask.
func (h *Handle) PushReservation(bytePos, bitPos int, release bool) {
	h.Lock()
	defer h.Unlock()
	h.Head = PushReservation(bytePos, bitPos, h.Head, release)
}

// GetFirstAvailable looks for the first unset bit in passed mask
func GetFirstAvailable(head *Sequence) (int, int, error) {
	byteIndex := 0
	current := head
	for current != nil {
		if current.Block != blockMAX {
			bytePos, bitPos := current.GetAvailableBit()
			return byteIndex + bytePos, bitPos, nil
		}
		byteIndex += int(current.Count * blockBytes)
		current = current.Next
	}
	return -1, -1, fmt.Errorf("no bit available")
}

// CheckIfAvailable checks if the bit correspondent to the specified ordinal is unset
// If the ordinal is beyond the Sequence limits, a negative response is returned
func CheckIfAvailable(head *Sequence, ordinal int) (int, int, error) {
	bytePos := ordinal / 8
	bitPos := ordinal % 8

	// Find the Sequence containing this byte
	current, _, _, inBlockBytePos := findSequence(head, bytePos)

	if current != nil {
		// Check whether the bit corresponding to the ordinal address is unset
		bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
		if current.Block&bitSel == 0 {
			return bytePos, bitPos, nil
		}
	}

	return -1, -1, fmt.Errorf("requested bit is not available")
}

// Given the byte position and the sequences list head, return the pointer to the
// sequence containing the byte (current), the pointer to the previous sequence,
// the number of blocks preceding the block containing the byte inside the current sequence.
// If bytePos is outside of the list, function will return (nil, nil, 0, -1)
func findSequence(head *Sequence, bytePos int) (*Sequence, *Sequence, uint32, int) {
	// Find the Sequence containing this byte
	previous := head
	current := head
	n := bytePos
	for current.Next != nil && n >= int(current.Count*blockBytes) { // Nil check for less than 32 addresses masks
		n -= int(current.Count * blockBytes)
		previous = current
		current = current.Next
	}

	// If byte is outside of the list, let caller know
	if n >= int(current.Count*blockBytes) {
		return nil, nil, 0, -1
	}

	// Find the byte position inside the block and the number of blocks
	// preceding the block containing the byte inside this sequence
	precBlocks := uint32(n / blockBytes)
	inBlockBytePos := bytePos % blockBytes

	return current, previous, precBlocks, inBlockBytePos
}

// PushReservation pushes the bit reservation inside the bitmask.
// Given byte and bit positions, identify the sequence (current) which holds the block containing the affected bit.
// Create a new block with the modified bit according to the operation (allocate/release).
// Create a new Sequence containing the new Block and insert it in the proper position.
// Remove current sequence if empty.
// Check if new Sequence can be merged with neighbour (previous/Next) sequences.
//
//
// Identify "current" Sequence containing block:
//                                      [prev seq] [current seq] [Next seq]
//
// Based on block position, resulting list of sequences can be any of three forms:
//
//        Block position                        Resulting list of sequences
// A) Block is first in current:         [prev seq] [new] [modified current seq] [Next seq]
// B) Block is last in current:          [prev seq] [modified current seq] [new] [Next seq]
// C) Block is in the middle of current: [prev seq] [curr pre] [new] [curr post] [Next seq]
func PushReservation(bytePos, bitPos int, head *Sequence, release bool) *Sequence {
	// Store list's head
	newHead := head

	// Find the Sequence containing this byte
	current, previous, precBlocks, inBlockBytePos := findSequence(head, bytePos)
	if current == nil {
		return newHead
	}

	// Construct updated block
	bitSel := uint32(blockFirstBit >> uint(inBlockBytePos*8+bitPos))
	newBlock := current.Block
	if release {
		newBlock &^= bitSel
	} else {
		newBlock |= bitSel
	}

	// Quit if it was a redundant request
	if current.Block == newBlock {
		return newHead
	}

	// Current Sequence inevitably looses one block, upadate Count
	current.Count--

	// Create new sequence
	newSequence := &Sequence{Block: newBlock, Count: 1}

	// Insert the new sequence in the list based on block position
	if precBlocks == 0 { // First in sequence (A)
		newSequence.Next = current
		if current == head {
			newHead = newSequence
			previous = newHead
		} else {
			previous.Next = newSequence
		}
		removeCurrentIfEmpty(&newHead, newSequence, current)
		mergeSequences(previous)
	} else if precBlocks == current.Count-2 { // Last in sequence (B)
		newSequence.Next = current.Next
		current.Next = newSequence
		mergeSequences(current)
	} else { // In between the sequence (C)
		currPre := &Sequence{Block: current.Block, Count: precBlocks, Next: newSequence}
		currPost := current
		currPost.Count -= precBlocks
		newSequence.Next = currPost
		if currPost == head {
			newHead = currPre
		} else {
			previous.Next = currPre
		}
		// No merging or empty current possible here
	}

	return newHead
}

// Removes the current sequence from the list if empty, adjusting the head pointer if needed
func removeCurrentIfEmpty(head **Sequence, previous, current *Sequence) {
	if current.Count == 0 {
		if current == *head {
			*head = current.Next
		} else {
			previous.Next = current.Next
			current = current.Next
		}
	}
}

// Given a pointer to a Sequence, it checks if it can be merged with any following sequences
// It stops when no more merging is possible.
// TODO: Optimization: only attempt merge from start to end sequence, no need to scan till the end of the list
func mergeSequences(seq *Sequence) {
	if seq != nil {
		// Merge all what possible from seq
		for seq.Next != nil && seq.Block == seq.Next.Block {
			seq.Count += seq.Next.Count
			seq.Next = seq.Next.Next
		}
		// Move to Next
		mergeSequences(seq.Next)
	}
}

// Serialize converts the sequence into a byte array
func Serialize(head *Sequence) ([]byte, error) {
	return nil, nil
}

// Deserialize decodes the byte array into a sequence
func Deserialize(data []byte) (*Sequence, error) {
	return nil, nil
}

func getNumBlocks(numBits uint32) uint32 {
	numBlocks := numBits / blockLen
	if numBits%blockLen != 0 {
		numBlocks++
	}
	return numBlocks
}
<|endoftext|>


<commit_before>package git

import (
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestRepository(t *testing.T) {
	path := os.TempDir()
	r := &Repository{
		path: path,
	}

	assert.Equal(t, path, r.Path())
}

func TestInit(t *testing.T) {
	tests := []struct {
		opt InitOptions
	}{
		{
			opt: InitOptions{},
		},
		{
			opt: InitOptions{
				Bare: true,
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			path := tempPath()
			defer func() {
				_ = os.RemoveAll(path)
			}()

			if err := Init(path, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestOpen(t *testing.T) {
	_, err := Open(testrepo.Path())
	assert.Nil(t, err)

	_, err = Open(tempPath())
	assert.Equal(t, os.ErrNotExist, err)
}

func TestClone(t *testing.T) {
	tests := []struct {
		opt CloneOptions
	}{
		{
			opt: CloneOptions{},
		},
		{
			opt: CloneOptions{
				Mirror: true,
				Bare:   true,
				Quiet:  true,
			},
		},
		{
			opt: CloneOptions{
				Branch: "develop",
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			path := tempPath()
			defer func() {
				_ = os.RemoveAll(path)
			}()

			if err := Clone(testrepo.Path(), path, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Fetch(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		opt FetchOptions
	}{
		{
			opt: FetchOptions{},
		},
		{
			opt: FetchOptions{
				Prune: true,
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Fetch(test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}
<commit_msg>repo: add more tests<commit_after>package git

import (
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestRepository(t *testing.T) {
	path := os.TempDir()
	r := &Repository{
		path: path,
	}

	assert.Equal(t, path, r.Path())
}

func TestInit(t *testing.T) {
	tests := []struct {
		opt InitOptions
	}{
		{
			opt: InitOptions{},
		},
		{
			opt: InitOptions{
				Bare: true,
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			path := tempPath()
			defer func() {
				_ = os.RemoveAll(path)
			}()

			if err := Init(path, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestOpen(t *testing.T) {
	_, err := Open(testrepo.Path())
	assert.Nil(t, err)

	_, err = Open(tempPath())
	assert.Equal(t, os.ErrNotExist, err)
}

func TestClone(t *testing.T) {
	tests := []struct {
		opt CloneOptions
	}{
		{
			opt: CloneOptions{},
		},
		{
			opt: CloneOptions{
				Mirror: true,
				Bare:   true,
				Quiet:  true,
			},
		},
		{
			opt: CloneOptions{
				Branch: "develop",
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			path := tempPath()
			defer func() {
				_ = os.RemoveAll(path)
			}()

			if err := Clone(testrepo.Path(), path, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Fetch(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		opt FetchOptions
	}{
		{
			opt: FetchOptions{},
		},
		{
			opt: FetchOptions{
				Prune: true,
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Fetch(test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Pull(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		opt PullOptions
	}{
		{
			opt: PullOptions{},
		},
		{
			opt: PullOptions{
				Rebase: true,
			},
		},
		{
			opt: PullOptions{
				All: true,
			},
		},
		{
			opt: PullOptions{
				Remote: "origin",
				Branch: "master",
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Pull(test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Push(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		remote string
		branch string
		opt    PushOptions
	}{
		{
			remote: "origin",
			branch: "master",
			opt:    PushOptions{},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Push(test.remote, test.branch, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Checkout(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		branch string
		opt    CheckoutOptions
	}{
		{
			branch: "develop",
			opt:    CheckoutOptions{},
		},
		{
			branch: "a-new-branch",
			opt: CheckoutOptions{
				BaseBranch: "master",
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Checkout(test.branch, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Reset(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		rev string
		opt ResetOptions
	}{
		{
			rev: "978fb7f6388b49b532fbef8b856681cfa6fcaa0a",
			opt: ResetOptions{
				Hard: true,
			},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Reset(test.rev, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}

func TestRepository_Move(t *testing.T) {
	path := tempPath()
	defer func() {
		_ = os.RemoveAll(path)
	}()

	if err := Clone(testrepo.Path(), path); err != nil {
		t.Fatal(err)
	}

	r, err := Open(path)
	if err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		src string
		dst string
		opt MoveOptions
	}{
		{
			src: "run.sh",
			dst: "runme.sh",
			opt: MoveOptions{},
		},
	}
	for _, test := range tests {
		t.Run("", func(t *testing.T) {
			// Make sure it does not blow up
			if err := r.Move(test.src, test.dst, test.opt); err != nil {
				t.Fatal(err)
			}
		})
	}
}
<|endoftext|>


<commit_before>package netutil

import (
	"bufio"

	"github.com/xiaonanln/goworld/engine/consts"
)

// BufferedWriteConnection provides buffered write to connections
type BufferedWriteConnection struct {
	Connection
	bufWriter *bufio.Writer
}

// NewBufferedWriteConnection creates a new connection with buffered write based on underlying connection
func NewBufferedWriteConnection(conn Connection) *BufferedWriteConnection {
	brc := &BufferedWriteConnection{
		Connection: conn,
	}
	brc.bufWriter = bufio.NewWriterSize(conn, consts.BUFFERED_WRITE_BUFFSIZE)
	return brc
}

func (brc *BufferedWriteConnection) Write(p []byte) (int, error) {
	return brc.bufWriter.Write(p)
}
<commit_msg>buffered write connection<commit_after>package netutil

import (
	"bufio"

	"github.com/xiaonanln/goworld/engine/consts"
)

// BufferedWriteConnection provides buffered write to connections
type BufferedWriteConnection struct {
	Connection
	bufWriter *bufio.Writer
}

// NewBufferedWriteConnection creates a new connection with buffered write based on underlying connection
func NewBufferedWriteConnection(conn Connection) *BufferedWriteConnection {
	brc := &BufferedWriteConnection{
		Connection: conn,
	}
	brc.bufWriter = bufio.NewWriterSize(conn, consts.BUFFERED_WRITE_BUFFSIZE)
	return brc
}

func (brc *BufferedWriteConnection) Write(p []byte) (int, error) {
	return brc.bufWriter.Write(p)
}

func (brc *BufferedWriteConnection) Flush() error {
	return brc.bufWriter.Flush()
}
<|endoftext|>


<commit_before>// 一个使用自定义评分规则搜索微博数据的例子
//
// 微博数据文件每行的格式是"<id>||||<timestamp>||||<uid>||||<reposts count>||||<text>"
// <timestamp>, <reposts count>和<text>的文本长度做评分数据
//
// 自定义评分规则为：
//	1. 首先排除关键词紧邻距离大于150个字节(五十个汉字)的微博
// 	2. 按照帖子距当前时间评分，精度为天，越晚的帖子评分越高
// 	3. 按照帖子BM25的整数部分排名
//	4. 同一天的微博再按照转发数评分，转发越多的帖子评分越高
//	5. 最后按照帖子长度评分，越长的帖子评分越高

package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"log"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/go-ego/riot"
	"github.com/go-ego/riot/types"
)

const (
	// SecondsInADay seconds in a day
	SecondsInADay = 86400
	// MaxTokenProximity max token proximity
	MaxTokenProximity = 150
)

var (
	weiboData = flag.String(
		"weibo_data",
		"../../testdata/weibo_data.txt",
		"索引的微博帖子，每行当作一个文档")
	query = flag.String(
		"query",
		"chinajoy游戏",
		"待搜索的短语")
	dictionaries = flag.String(
		"dictionaries",
		"../../data/dict/dictionary.txt",
		"分词字典文件")
	stopTokenFile = flag.String(
		"stop_token_file",
		"../../data/dict/stop_tokens.txt",
		"停用词文件")

	searcher = riot.Engine{}
	options  = types.RankOptions{
		ScoringCriteria: WeiboScoringCriteria{},
		OutputOffset:    0,
		MaxOutputs:      100,
	}
	searchQueries = []string{}
)

// WeiboScoringFields 微博评分字段
type WeiboScoringFields struct {
	// 帖子的时间戳
	Timestamp uint32

	// 帖子的转发数
	RepostsCount uint32

	// 帖子的长度
	TextLength int
}

// WeiboScoringCriteria 自定义的微博评分规则
type WeiboScoringCriteria struct {
}

// Score score and sort
func (criteria WeiboScoringCriteria) Score(
	doc types.IndexedDocument, fields interface{}) []float32 {
	if doc.TokenProximity > MaxTokenProximity { // 评分第一步
		return []float32{}
	}
	if reflect.TypeOf(fields) != reflect.TypeOf(WeiboScoringFields{}) {
		return []float32{}
	}
	output := make([]float32, 4)
	wsf := fields.(WeiboScoringFields)
	output[0] = float32(wsf.Timestamp / SecondsInADay) // 评分第二步
	output[1] = float32(int(doc.BM25))                 // 评分第三步
	output[2] = float32(wsf.RepostsCount)              // 评分第四步
	output[3] = float32(wsf.TextLength)                // 评分第五步
	return output
}

func main() {
	// 解析命令行参数
	flag.Parse()
	log.Printf("待搜索的短语为\"%s\"", *query)

	// 初始化
	gob.Register(WeiboScoringFields{})
	searcher.Init(types.EngineInitOptions{
		SegmenterDict: *dictionaries,
		StopTokenFile: *stopTokenFile,
		IndexerInitOptions: &types.IndexerInitOptions{
			IndexType: types.LocationsIndex,
		},
		DefaultRankOptions: &options,
	})
	defer searcher.Close()

	// 读入微博数据
	file, err := os.Open(*weiboData)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	log.Printf("读入文本 %s", *weiboData)
	scanner := bufio.NewScanner(file)
	lines := []string{}
	fieldsSlice := []WeiboScoringFields{}
	for scanner.Scan() {
		data := strings.Split(scanner.Text(), "||||")
		if len(data) != 10 {
			continue
		}
		timestamp, _ := strconv.ParseUint(data[1], 10, 32)
		repostsCount, _ := strconv.ParseUint(data[4], 10, 32)
		text := data[9]
		if text != "" {
			lines = append(lines, text)
			fields := WeiboScoringFields{
				Timestamp:    uint32(timestamp),
				RepostsCount: uint32(repostsCount),
				TextLength:   len(text),
			}
			fieldsSlice = append(fieldsSlice, fields)
		}
	}
	log.Printf("读入%d条微博\n", len(lines))

	// 建立索引
	log.Print("建立索引")
	for i, text := range lines {
		searcher.IndexDocument(uint64(i),
			types.DocIndexData{Content: text, Fields: fieldsSlice[i]}, false)
	}
	searcher.FlushIndex()
	log.Print("索引建立完毕")

	// 搜索
	log.Printf("开始查询")
	output := searcher.Search(types.SearchRequest{Text: *query})

	// 显示
	fmt.Println()
	for _, doc := range output.Docs {
		fmt.Printf("%v %s\n\n", doc.Scores, lines[doc.DocId])
	}
	log.Printf("查询完毕")
}
<commit_msg>Update and fix example<commit_after>// 一个使用自定义评分规则搜索微博数据的例子
//
// 微博数据文件每行的格式是"<id>||||<timestamp>||||<uid>||||<reposts count>||||<text>"
// <timestamp>, <reposts count>和<text>的文本长度做评分数据
//
// 自定义评分规则为：
//	1. 首先排除关键词紧邻距离大于150个字节(五十个汉字)的微博
// 	2. 按照帖子距当前时间评分，精度为天，越晚的帖子评分越高
// 	3. 按照帖子BM25的整数部分排名
//	4. 同一天的微博再按照转发数评分，转发越多的帖子评分越高
//	5. 最后按照帖子长度评分，越长的帖子评分越高

package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"log"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/go-ego/riot"
	"github.com/go-ego/riot/types"
)

const (
	// SecondsInADay seconds in a day
	SecondsInADay = 86400
	// MaxTokenProximity max token proximity
	MaxTokenProximity = 150
)

var (
	weiboData = flag.String(
		"weibo_data",
		"../../testdata/weibo_data.txt",
		"索引的微博帖子，每行当作一个文档")
	query = flag.String(
		"query",
		"chinajoy 游戏",
		"待搜索的短语")
	dictionaries = flag.String(
		"dictionaries",
		"../../data/dict/dictionary.txt",
		"分词字典文件")
	stopTokenFile = flag.String(
		"stop_token_file",
		"../../data/dict/stop_tokens.txt",
		"停用词文件")

	searcher = riot.Engine{}
	options  = types.RankOptions{
		ScoringCriteria: WeiboScoringCriteria{},
		OutputOffset:    0,
		MaxOutputs:      100,
	}
	searchQueries = []string{}
)

// WeiboScoringFields 微博评分字段
type WeiboScoringFields struct {
	// 帖子的时间戳
	Timestamp uint32

	// 帖子的转发数
	RepostsCount uint32

	// 帖子的长度
	TextLength int
}

// WeiboScoringCriteria 自定义的微博评分规则
type WeiboScoringCriteria struct {
}

// Score score and sort
func (criteria WeiboScoringCriteria) Score(
	doc types.IndexedDocument, fields interface{}) []float32 {
	if doc.TokenProximity > MaxTokenProximity { // 评分第一步
		return []float32{}
	}
	if reflect.TypeOf(fields) != reflect.TypeOf(WeiboScoringFields{}) {
		return []float32{}
	}
	output := make([]float32, 4)
	wsf := fields.(WeiboScoringFields)
	output[0] = float32(wsf.Timestamp / SecondsInADay) // 评分第二步
	output[1] = float32(int(doc.BM25))                 // 评分第三步
	output[2] = float32(wsf.RepostsCount)              // 评分第四步
	output[3] = float32(wsf.TextLength)                // 评分第五步
	return output
}

func main() {
	// 解析命令行参数
	flag.Parse()
	log.Printf("待搜索的短语为\"%s\"", *query)

	// 初始化
	gob.Register(WeiboScoringFields{})
	searcher.Init(types.EngineInitOptions{
		Using:         1,
		SegmenterDict: *dictionaries,
		StopTokenFile: *stopTokenFile,
		IndexerInitOptions: &types.IndexerInitOptions{
			IndexType: types.LocationsIndex,
		},
		DefaultRankOptions: &options,
	})
	defer searcher.Close()

	// 读入微博数据
	file, err := os.Open(*weiboData)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	log.Printf("读入文本 %s", *weiboData)
	scanner := bufio.NewScanner(file)
	lines := []string{}
	fieldsSlice := []WeiboScoringFields{}
	for scanner.Scan() {
		data := strings.Split(scanner.Text(), "||||")
		if len(data) != 10 {
			continue
		}
		timestamp, _ := strconv.ParseUint(data[1], 10, 32)
		repostsCount, _ := strconv.ParseUint(data[4], 10, 32)
		text := data[9]
		if text != "" {
			lines = append(lines, text)
			fields := WeiboScoringFields{
				Timestamp:    uint32(timestamp),
				RepostsCount: uint32(repostsCount),
				TextLength:   len(text),
			}
			fieldsSlice = append(fieldsSlice, fields)
		}
	}
	log.Printf("读入%d条微博\n", len(lines))

	// 建立索引
	log.Print("建立索引")
	for i, text := range lines {
		searcher.IndexDocument(uint64(i),
			types.DocIndexData{Content: text, Fields: fieldsSlice[i]}, false)
	}
	searcher.FlushIndex()
	log.Print("索引建立完毕")

	// 搜索
	log.Printf("开始查询")
	output := searcher.Search(types.SearchRequest{Text: *query})

	// 显示
	fmt.Println("output...")
	for _, doc := range output.Docs {
		fmt.Printf("%v %s\n\n", doc.Scores, lines[doc.DocId])
	}
	log.Printf("查询完毕")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

// EC2Plugin is a mackerel plugin for ec2
type EC2Plugin struct {
	InstanceID  string
	Region      string
	Credentials *credentials.Credentials
	CloudWatch  *cloudwatch.CloudWatch
}

var graphdef = map[string](mp.Graphs){
	"ec2.CPUUtilization": mp.Graphs{
		Label: "CPU Utilization",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"ec2.DiskBytes": mp.Graphs{
		Label: "Disk Bytes",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "DiskReadBytes", Label: "DiskReadBytes"},
			mp.Metrics{Name: "DiskWriteBytes", Label: "DiskWriteBytes"},
		},
	},
	"ec2.DiskOps": mp.Graphs{
		Label: "Disk Ops",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "DiskReadOps", Label: "DiskReadOps"},
			mp.Metrics{Name: "DiskWriteOps", Label: "DiskWriteOps"},
		},
	},
	"ec2.Network": mp.Graphs{
		Label: "Network",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "NetworkIn", Label: "NetworkIn"},
			mp.Metrics{Name: "NetworkOut", Label: "NetworkOut"},
		},
	},
	"ec2.NetworkPackets": mp.Graphs{
		Label: "Network",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "NetworkPacketsIn", Label: "NetworkPacketsIn"},
			mp.Metrics{Name: "NetworkPacketsOut", Label: "NetworkPacketsOut"},
		},
	},
	"ec2.StatusCheckFailed": mp.Graphs{
		Label: "StatusCheckFailed",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "StatusCheckFailed", Label: "StatusCheckFailed"},
			mp.Metrics{Name: "StatusCheckFailed_Instance", Label: "StatusCheckFailed_Instance"},
			mp.Metrics{Name: "StatusCheckFailed_System", Label: "StatusCheckFailed_System"},
		},
	},
}

// GraphDefinition returns graphdef
func (p EC2Plugin) GraphDefinition() map[string](mp.Graphs) {
	return graphdef
}

func getLastPoint(cloudWatch *cloudwatch.CloudWatch, dimension *cloudwatch.Dimension, metricName string) (float64, error) {
	now := time.Now()
	response, err := cloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		StartTime:  aws.Time(now.Add(time.Duration(600) * time.Second * -1)),
		EndTime:    aws.Time(now),
		Period:     aws.Int64(60),
		Namespace:  aws.String("AWS/EC2"),
		Dimensions: []*cloudwatch.Dimension{dimension},
		MetricName: aws.String(metricName),
		Statistics: []*string{aws.String("Average")},
	})
	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := time.Unix(0, 0)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(latest) {
			continue
		}

		latest = *dp.Timestamp
		latestVal = *dp.Average
	}

	return latestVal, nil
}

// FetchMetrics fetches metrics from CloudWatch
func (p EC2Plugin) FetchMetrics() (map[string]float64, error) {
	stat := make(map[string]float64)
	p.CloudWatch = cloudwatch.New(session.New(
		&aws.Config{
			Credentials: p.Credentials,
			Region:      &p.Region,
		}))
	dimension := &cloudwatch.Dimension{
		Name:  aws.String("InstanceId"),
		Value: aws.String(p.InstanceID),
	}

	for _, met := range [...]string{
		"CPUUtilization",
		"DiskReadBytes",
		"DiskReadOps",
		"DiskWriteBytes",
		"DiskWriteOps",
		"NetworkIn",
		"NetworkOut",
		"NetworkPacketsIn",
		"NetworkPacketsOut",
		"StatusCheckFailed",
		"StatusCheckFailed_Instance",
		"StatusCheckFailed_System",
	} {
		v, err := getLastPoint(p.CloudWatch, dimension, met)
		if err == nil {
			stat[met] = v
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	return stat, nil
}

func main() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optInstanceID := flag.String("instance-id", "", "Instance ID")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ec2 EC2Plugin

	// use credentials from option
	if *optAccessKeyID != "" && *optSecretAccessKey != "" {
		ec2.Credentials = credentials.NewStaticCredentials(*optAccessKeyID, *optSecretAccessKey, "")
	}

	// get metadata in ec2 instance
	ec2MC := ec2metadata.New(session.New())
	ec2.Region = *optRegion
	if *optRegion == "" {
		ec2.Region, _ = ec2MC.Region()
	}
	ec2.InstanceID = *optInstanceID
	if *optInstanceID == "" {
		ec2.InstanceID, _ = ec2MC.GetMetadata("instance-id")
	}

	helper := mp.NewMackerelPlugin(ec2)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-aws-ec2-%s", ec2.InstanceID)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>gofmt -s<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

// EC2Plugin is a mackerel plugin for ec2
type EC2Plugin struct {
	InstanceID  string
	Region      string
	Credentials *credentials.Credentials
	CloudWatch  *cloudwatch.CloudWatch
}

var graphdef = map[string]mp.Graphs{
	"ec2.CPUUtilization": {
		Label: "CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"ec2.DiskBytes": {
		Label: "Disk Bytes",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "DiskReadBytes", Label: "DiskReadBytes"},
			{Name: "DiskWriteBytes", Label: "DiskWriteBytes"},
		},
	},
	"ec2.DiskOps": {
		Label: "Disk Ops",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "DiskReadOps", Label: "DiskReadOps"},
			{Name: "DiskWriteOps", Label: "DiskWriteOps"},
		},
	},
	"ec2.Network": {
		Label: "Network",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "NetworkIn", Label: "NetworkIn"},
			{Name: "NetworkOut", Label: "NetworkOut"},
		},
	},
	"ec2.NetworkPackets": {
		Label: "Network",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "NetworkPacketsIn", Label: "NetworkPacketsIn"},
			{Name: "NetworkPacketsOut", Label: "NetworkPacketsOut"},
		},
	},
	"ec2.StatusCheckFailed": {
		Label: "StatusCheckFailed",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "StatusCheckFailed", Label: "StatusCheckFailed"},
			{Name: "StatusCheckFailed_Instance", Label: "StatusCheckFailed_Instance"},
			{Name: "StatusCheckFailed_System", Label: "StatusCheckFailed_System"},
		},
	},
}

// GraphDefinition returns graphdef
func (p EC2Plugin) GraphDefinition() map[string]mp.Graphs {
	return graphdef
}

func getLastPoint(cloudWatch *cloudwatch.CloudWatch, dimension *cloudwatch.Dimension, metricName string) (float64, error) {
	now := time.Now()
	response, err := cloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		StartTime:  aws.Time(now.Add(time.Duration(600) * time.Second * -1)),
		EndTime:    aws.Time(now),
		Period:     aws.Int64(60),
		Namespace:  aws.String("AWS/EC2"),
		Dimensions: []*cloudwatch.Dimension{dimension},
		MetricName: aws.String(metricName),
		Statistics: []*string{aws.String("Average")},
	})
	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := time.Unix(0, 0)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(latest) {
			continue
		}

		latest = *dp.Timestamp
		latestVal = *dp.Average
	}

	return latestVal, nil
}

// FetchMetrics fetches metrics from CloudWatch
func (p EC2Plugin) FetchMetrics() (map[string]float64, error) {
	stat := make(map[string]float64)
	p.CloudWatch = cloudwatch.New(session.New(
		&aws.Config{
			Credentials: p.Credentials,
			Region:      &p.Region,
		}))
	dimension := &cloudwatch.Dimension{
		Name:  aws.String("InstanceId"),
		Value: aws.String(p.InstanceID),
	}

	for _, met := range [...]string{
		"CPUUtilization",
		"DiskReadBytes",
		"DiskReadOps",
		"DiskWriteBytes",
		"DiskWriteOps",
		"NetworkIn",
		"NetworkOut",
		"NetworkPacketsIn",
		"NetworkPacketsOut",
		"StatusCheckFailed",
		"StatusCheckFailed_Instance",
		"StatusCheckFailed_System",
	} {
		v, err := getLastPoint(p.CloudWatch, dimension, met)
		if err == nil {
			stat[met] = v
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	return stat, nil
}

func main() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optInstanceID := flag.String("instance-id", "", "Instance ID")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ec2 EC2Plugin

	// use credentials from option
	if *optAccessKeyID != "" && *optSecretAccessKey != "" {
		ec2.Credentials = credentials.NewStaticCredentials(*optAccessKeyID, *optSecretAccessKey, "")
	}

	// get metadata in ec2 instance
	ec2MC := ec2metadata.New(session.New())
	ec2.Region = *optRegion
	if *optRegion == "" {
		ec2.Region, _ = ec2MC.Region()
	}
	ec2.InstanceID = *optInstanceID
	if *optInstanceID == "" {
		ec2.InstanceID, _ = ec2MC.GetMetadata("instance-id")
	}

	helper := mp.NewMackerelPlugin(ec2)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-aws-ec2-%s", ec2.InstanceID)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>