<commit_before>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hook

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"context"
	"github.com/goharbor/harbor/src/jobservice/common/utils"
)

const (
	proxyEnvHTTP  = "http_proxy"
	proxyEnvHTTPS = "https_proxy"
)

// Client for handling the hook events
type Client interface {
	// SendEvent send the event to the subscribed parties
	SendEvent(evt *Event) error
}

// Client is used to post the related data to the interested parties.
type basicClient struct {
	client *http.Client
	ctx    context.Context
}

// NewClient return the ptr of the new hook client
func NewClient(ctx context.Context) Client {
	// Create transport
	transport := &http.Transport{
		MaxIdleConns:    20,
		IdleConnTimeout: 30 * time.Second,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		TLSHandshakeTimeout:   10 * time.Second,
		ResponseHeaderTimeout: 10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}

	// Get the http/https proxies
	proxyAddr, ok := os.LookupEnv(proxyEnvHTTP)
	if !ok {
		proxyAddr, ok = os.LookupEnv(proxyEnvHTTPS)
	}

	if ok && !utils.IsEmptyStr(proxyAddr) {
		proxyURL, err := url.Parse(proxyAddr)
		if err == nil {
			transport.Proxy = http.ProxyURL(proxyURL)
		}
	}

	client := &http.Client{
		Timeout:   15 * time.Second,
		Transport: transport,
	}

	return &basicClient{
		client: client,
		ctx:    ctx,
	}
}

// ReportStatus reports the status change info to the subscribed party.
// The status includes 'checkin' info with format 'check_in:<message>'
func (bc *basicClient) SendEvent(evt *Event) error {
	if evt == nil {
		return errors.New("nil event")
	}

	if err := evt.Validate(); err != nil {
		return err
	}

	// Marshal data
	data, err := json.Marshal(evt.Data)
	if err != nil {
		return err
	}

	// New post request
	req, err := http.NewRequest(http.MethodPost, evt.URL, strings.NewReader(string(data)))
	if err != nil {
		return err
	}

	res, err := bc.client.Do(req)
	if err != nil {
		return err
	}

	defer func() {
		_ = res.Body.Close()
	}() // close connection for reuse

	// Should be 200
	if res.StatusCode != http.StatusOK {
		if res.ContentLength > 0 {
			// read error content and return
			dt, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return err
			}
			return errors.New(string(dt))
		}

		return fmt.Errorf("failed to report status change via hook, expect '200' but got '%d'", res.StatusCode)
	}

	return nil
}
<commit_msg>Make jobservice use the proxy env variables (including no_proxy)<commit_after>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hook

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"context"
)

// Client for handling the hook events
type Client interface {
	// SendEvent send the event to the subscribed parties
	SendEvent(evt *Event) error
}

// Client is used to post the related data to the interested parties.
type basicClient struct {
	client *http.Client
	ctx    context.Context
}

// NewClient return the ptr of the new hook client
func NewClient(ctx context.Context) Client {
	// Create transport
	transport := &http.Transport{
		MaxIdleConns:    20,
		IdleConnTimeout: 30 * time.Second,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		TLSHandshakeTimeout:   10 * time.Second,
		ResponseHeaderTimeout: 10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		Proxy:                 http.ProxyFromEnvironment,
	}

	client := &http.Client{
		Timeout:   15 * time.Second,
		Transport: transport,
	}

	return &basicClient{
		client: client,
		ctx:    ctx,
	}
}

// ReportStatus reports the status change info to the subscribed party.
// The status includes 'checkin' info with format 'check_in:<message>'
func (bc *basicClient) SendEvent(evt *Event) error {
	if evt == nil {
		return errors.New("nil event")
	}

	if err := evt.Validate(); err != nil {
		return err
	}

	// Marshal data
	data, err := json.Marshal(evt.Data)
	if err != nil {
		return err
	}

	// New post request
	req, err := http.NewRequest(http.MethodPost, evt.URL, strings.NewReader(string(data)))
	if err != nil {
		return err
	}

	res, err := bc.client.Do(req)
	if err != nil {
		return err
	}

	defer func() {
		_ = res.Body.Close()
	}() // close connection for reuse

	// Should be 200
	if res.StatusCode != http.StatusOK {
		if res.ContentLength > 0 {
			// read error content and return
			dt, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return err
			}
			return errors.New(string(dt))
		}

		return fmt.Errorf("failed to report status change via hook, expect '200' but got '%d'", res.StatusCode)
	}

	return nil
}
<|endoftext|>


<commit_before>package amalog // import "github.com/amalog/go"

import (
	"os"

	"github.com/amalog/go/term"
)

type Machine struct {
	root term.Term // term which started it all
}

func NewMachine() *Machine {
	return &Machine{}
}

// LoadRoot populates the machine's root term with the contents of a file
// and loads all of that term's module dependencies.
func (m *Machine) LoadRoot(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	m.root, err = term.ReadAll(file)
	if err != nil {
		return err
	}

	return nil
}

func (m *Machine) Call(name string, args ...term.Term) *Result {
	return nil
}
<commit_msg>Implement module search path<commit_after>package amalog // import "github.com/amalog/go"

import (
	"os"
	"path/filepath"

	"github.com/amalog/go/term"
)

type Machine struct {
	root term.Term // term which started it all
}

func NewMachine() *Machine {
	return &Machine{}
}

// LoadRoot populates the machine's root term with the contents of a file
// and loads all of that term's module dependencies.
func (m *Machine) LoadRoot(filename string) error {
	// read root term from file
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	m.root, err = term.ReadAll(file)
	if err != nil {
		return err
	}

	// find directories to search for modules
	modulePath, err := m.modulePath(filename)
	if err != nil {
		return err
	}
	_ = modulePath

	return nil
}

func (m *Machine) Call(name string, args ...term.Term) *Result {
	return nil
}

// modulePath returns a list of directories within which we should search for
// modules. The directories returned are from highest to lowest priority.  All
// directories are guaranteed to exist in the filesystem at the time this method
// was invoked.
//
// The argument should be the path to source code for the root term.
func (m *Machine) modulePath(src string) ([]string, error) {
	// start search at directories holding source code and executable
	starts := make([]string, 0, 2)
	for _, p := range []string{src, os.Args[0]} {
		p, err := filepath.Abs(p)
		if err != nil {
			return nil, err
		}
		starts = append(starts, filepath.Dir(p))
	}

	// walk back to the root looking for module directories
	paths := make([]string, 0)
	tried := make(map[string]bool)
	for _, p := range starts {
		for {
			tried[p] = true
			candidate := filepath.Join(p, "amalog_modules")
			if stat, err := os.Stat(candidate); err == nil && stat.IsDir() {
				paths = append(paths, candidate)
			} else if err != nil && !os.IsNotExist(err) {
				return nil, err // unexpected error
			}

			p = filepath.Dir(p)
			if tried[p] {
				break
			}
		}
	}

	return paths, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"geo"
	"graph"
	"kdtree"
	"log"
	"mm"
	"os"
	"path"
	"runtime"
	"sort"
)

const (
	MaxThreads = 8
	BBoxMargin = 0.002
)

var (
	FlagBaseDir string
)

func init() {
	flag.StringVar(&FlagBaseDir, "dir", "", "directory of the graph files")
}

func main() {
	runtime.GOMAXPROCS(MaxThreads)
	flag.Parse()

	clusterGraph, err := graph.OpenClusterGraph(FlagBaseDir, false /* loadMatrices */)
	if err != nil {
		log.Fatal("Loading graph: ", err)
	}

	fmt.Printf("Create k-d trees for the subgraphs\n")
	bboxes := make([]geo.BBox, len(clusterGraph.Cluster))
	ready := make(chan int, len(clusterGraph.Cluster))
	for i, g := range clusterGraph.Cluster {
		clusterDir := fmt.Sprintf("/cluster%d", i+1)
		go writeKdTreeSubgraph(ready, path.Join(FlagBaseDir, clusterDir), g.(*graph.GraphFile), bboxes, i)
	}
	for _, _ = range clusterGraph.Cluster {
		<-ready
	}

	// write bounding boxes to file
	fmt.Printf("Write bounding boxes\n")
	var bboxesFile []int32
	err = mm.Create(path.Join(FlagBaseDir, "bboxes.ftf"), len(bboxes)*4, &bboxesFile)
	if err != nil {
		log.Fatal("mm.Create failed: ", err)
	}
	for i, b := range bboxes {
		encodedBox := b.Encode()
		for j := 0; j < 4; j++ {
			bboxesFile[4*i+j] = encodedBox[j]
		}
	}
	err = mm.Close(&bboxesFile)
	if err != nil {
		log.Fatal("mm.Close failed: ", err)
	}

	fmt.Printf("Create k-d trees for the overlay graph\n")
	writeKdTreeOverlay(path.Join(FlagBaseDir, "/overlay"), clusterGraph.Overlay.(*graph.OverlayGraphFile))
}

type byLat struct {
	*kdtree.KdTree
}

func (x byLat) Less(i, j int) bool {
	return x.KdTree.Coordinates[i].Lat < x.KdTree.Coordinates[j].Lat
}

type byLng struct {
	*kdtree.KdTree
}

func (x byLng) Less(i, j int) bool {
	return x.KdTree.Coordinates[i].Lng < x.KdTree.Coordinates[j].Lng
}

func createKdTreeSubgraph(g *graph.GraphFile) (*kdtree.KdTree, geo.BBox) {
	estimatedSize := g.VertexCount() + 2*g.EdgeCount()
	encodedSteps := make([]uint64, 0, estimatedSize)
	coordinates := make([]geo.Coordinate, 0, estimatedSize)

	bbox := geo.NewBBoxPoint(g.VertexCoordinate(graph.Vertex(0)))

	t := &kdtree.KdTree{Graph: g, EncodedSteps: encodedSteps, Coordinates: coordinates}

	// line up all coordinates and their encodings in the graph
	edges := []graph.Edge(nil)
	for i := 0; i < g.VertexCount(); i++ {
		vertex := graph.Vertex(i)
		coordinates = append(coordinates, g.VertexCoordinate(vertex))
		t.AppendEncodedStep(encodeCoordinate(i, kdtree.MaxEdgeOffset, kdtree.MaxStepOffset))
		bbox.Union(geo.NewBBoxPoint(g.VertexCoordinate(vertex)))

		edges = g.VertexRawEdges(vertex, edges)
		for j, e := range edges {
			steps := g.EdgeSteps(e, vertex)

			if len(steps) > 2000 {
				panic("steps > 2000")
			}

			for k, s := range steps {
				coordinates = append(coordinates, s)
				t.AppendEncodedStep(encodeCoordinate(i, j, k))
				bbox.Union(geo.NewBBoxPoint(s))
			}
		}
	}
	createSubTree(t, true)
	return t, bbox
}

func createKdTreeOverlay(g *graph.OverlayGraphFile) *kdtree.KdTree {
	estimatedSize := g.VertexCount() + 2*g.EdgeCount()
	encodedSteps := make([]uint64, 0, estimatedSize)
	coordinates := make([]geo.Coordinate, 0, estimatedSize)

	t := &kdtree.KdTree{Graph: g, EncodedSteps: encodedSteps, Coordinates: coordinates}

	// line up all coordinates and their encodings in the graph
	edges := []graph.Edge(nil)
	fmt.Printf("Vertex count: %d\n", g.VertexCount())
	for i := 0; i < g.VertexCount(); i++ {
		vertex := graph.Vertex(i)
		coordinates = append(coordinates, g.VertexCoordinate(vertex))
		t.AppendEncodedStep(encodeCoordinate(i, kdtree.MaxEdgeOffset, kdtree.MaxStepOffset))

		edges = g.VertexRawEdges(vertex, edges)
		for j, e := range edges {
			steps := g.EdgeSteps(e, vertex)
			for k, s := range steps {
				coordinates = append(coordinates, s)
				t.AppendEncodedStep(encodeCoordinate(i, j, k))
			}
		}
	}

	createSubTree(t, true)
	return t
}

func subKdTree(t *kdtree.KdTree, from, to int) *kdtree.KdTree {
	// The EncodedSteps slice is restricted by pointers and not with a new slice due to its encoding.
	return &kdtree.KdTree{Graph: t.Graph, EncodedSteps: t.EncodedSteps, Coordinates: t.Coordinates[from:to],
		EncodedStepsStart: t.EncodedStepsStart + from, EncodedStepsEnd: t.EncodedStepsStart + to - 1}
}

func createSubTree(t *kdtree.KdTree, compareLat bool) {
	if t.Len() <= 1 {
		return
	}
	if compareLat {
		sort.Sort(byLat{t})
	} else {
		sort.Sort(byLng{t})
	}
	middle := t.Len() / 2
	createSubTree(subKdTree(t, 0, middle), !compareLat)
	createSubTree(subKdTree(t, middle+1, t.Len()), !compareLat)
}

// writeKdTreeSubgraph creates and stores the k-d tree for the given cluster graph
func writeKdTreeSubgraph(ready chan<- int, baseDir string, g *graph.GraphFile, bboxes []geo.BBox, pos int) {
	t, bbox := createKdTreeSubgraph(g)
	err := writeToFile(t, baseDir)
	if err != nil {
		log.Fatal("Creating k-d tree: ", err)
	}
	// a very simple margin is added
	bbox.Min.Lat -= BBoxMargin
	bbox.Min.Lng -= BBoxMargin
	bbox.Max.Lat += BBoxMargin
	bbox.Max.Lng += BBoxMargin

	bboxes[pos] = bbox
	ready <- 1
}

// writeKdTree creates and stores the k-d tree for the given graph
func writeKdTreeOverlay(baseDir string, g *graph.OverlayGraphFile) {
	t := createKdTreeOverlay(g)
	err := writeToFile(t, baseDir)
	if err != nil {
		log.Fatal("Creating k-d tree: ", err)
	}
}

// writeToFile stores the permitation created by the k-d tree
func writeToFile(t *kdtree.KdTree, baseDir string) error {
	output, err := os.Create(path.Join(baseDir, "kdtree.ftf"))
	defer output.Close()
	if err != nil {
		return err
	}
	writeErr := binary.Write(output, binary.LittleEndian, t.EncodedSteps)
	if writeErr != nil {
		return writeErr
	}
	return nil
}

func encodeCoordinate(vertexIndex, edgeOffset, stepOffset int) uint64 {
	if vertexIndex > kdtree.MaxVertexIndex {
		panic("vertex index too large")
	}
	// both offsets are at max if only a vertex is encoded
	if edgeOffset != kdtree.MaxEdgeOffset && stepOffset != kdtree.MaxStepOffset {
		if edgeOffset >= kdtree.MaxEdgeOffset {
			panic("edge offset too large")
		}
		if stepOffset >= kdtree.MaxStepOffset {
			fmt.Printf("vertex: %d, edge offset: %v, step offset: %v\n", vertexIndex, edgeOffset, stepOffset)
			panic("step offset too large")
		}
	}

	ec := uint64(vertexIndex) << (kdtree.EdgeOffsetBits + kdtree.StepOffsetBits)
	ec |= uint64(edgeOffset) << kdtree.StepOffsetBits
	ec |= uint64(stepOffset)
	return ec
}
<commit_msg>fixed bounding box computation<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"geo"
	"graph"
	"kdtree"
	"log"
	"mm"
	"os"
	"path"
	"runtime"
	"sort"
)

const (
	MaxThreads = 8
	BBoxMargin = 0.002
)

var (
	FlagBaseDir string
)

func init() {
	flag.StringVar(&FlagBaseDir, "dir", "", "directory of the graph files")
}

func main() {
	runtime.GOMAXPROCS(MaxThreads)
	flag.Parse()

	clusterGraph, err := graph.OpenClusterGraph(FlagBaseDir, false /* loadMatrices */)
	if err != nil {
		log.Fatal("Loading graph: ", err)
	}

	fmt.Printf("Create k-d trees for the subgraphs\n")
	bboxes := make([]geo.BBox, len(clusterGraph.Cluster))
	ready := make(chan int, len(clusterGraph.Cluster))
	for i, g := range clusterGraph.Cluster {
		clusterDir := fmt.Sprintf("/cluster%d", i+1)
		go writeKdTreeSubgraph(ready, path.Join(FlagBaseDir, clusterDir), g.(*graph.GraphFile), bboxes, i)
	}
	for _, _ = range clusterGraph.Cluster {
		<-ready
	}

	// write bounding boxes to file
	fmt.Printf("Write bounding boxes\n")
	var bboxesFile []int32
	err = mm.Create(path.Join(FlagBaseDir, "bboxes.ftf"), len(bboxes)*4, &bboxesFile)
	if err != nil {
		log.Fatal("mm.Create failed: ", err)
	}
	for i, b := range bboxes {
		encodedBox := b.Encode()
		for j := 0; j < 4; j++ {
			bboxesFile[4*i+j] = encodedBox[j]
		}
	}
	err = mm.Close(&bboxesFile)
	if err != nil {
		log.Fatal("mm.Close failed: ", err)
	}

	fmt.Printf("Create k-d trees for the overlay graph\n")
	writeKdTreeOverlay(path.Join(FlagBaseDir, "/overlay"), clusterGraph.Overlay.(*graph.OverlayGraphFile))
}

type byLat struct {
	*kdtree.KdTree
}

func (x byLat) Less(i, j int) bool {
	return x.KdTree.Coordinates[i].Lat < x.KdTree.Coordinates[j].Lat
}

type byLng struct {
	*kdtree.KdTree
}

func (x byLng) Less(i, j int) bool {
	return x.KdTree.Coordinates[i].Lng < x.KdTree.Coordinates[j].Lng
}

func createKdTreeSubgraph(g *graph.GraphFile) (*kdtree.KdTree, geo.BBox) {
	estimatedSize := g.VertexCount() + 2*g.EdgeCount()
	encodedSteps := make([]uint64, 0, estimatedSize)
	coordinates := make([]geo.Coordinate, 0, estimatedSize)

	bbox := geo.NewBBoxPoint(g.VertexCoordinate(graph.Vertex(0)))

	t := &kdtree.KdTree{Graph: g, EncodedSteps: encodedSteps, Coordinates: coordinates}

	// line up all coordinates and their encodings in the graph
	edges := []graph.Edge(nil)
	for i := 0; i < g.VertexCount(); i++ {
		vertex := graph.Vertex(i)
		coordinates = append(coordinates, g.VertexCoordinate(vertex))
		t.AppendEncodedStep(encodeCoordinate(i, kdtree.MaxEdgeOffset, kdtree.MaxStepOffset))
		bbox = bbox.Union(geo.NewBBoxPoint(g.VertexCoordinate(vertex)))

		edges = g.VertexRawEdges(vertex, edges)
		for j, e := range edges {
			steps := g.EdgeSteps(e, vertex)

			if len(steps) > 2000 {
				panic("steps > 2000")
			}

			for k, s := range steps {
				coordinates = append(coordinates, s)
				t.AppendEncodedStep(encodeCoordinate(i, j, k))
				bbox = bbox.Union(geo.NewBBoxPoint(s))
			}
		}
	}
	createSubTree(t, true)
	return t, bbox
}

func createKdTreeOverlay(g *graph.OverlayGraphFile) *kdtree.KdTree {
	estimatedSize := g.VertexCount() + 2*g.EdgeCount()
	encodedSteps := make([]uint64, 0, estimatedSize)
	coordinates := make([]geo.Coordinate, 0, estimatedSize)

	t := &kdtree.KdTree{Graph: g, EncodedSteps: encodedSteps, Coordinates: coordinates}

	// line up all coordinates and their encodings in the graph
	edges := []graph.Edge(nil)
	fmt.Printf("Vertex count: %d\n", g.VertexCount())
	for i := 0; i < g.VertexCount(); i++ {
		vertex := graph.Vertex(i)
		coordinates = append(coordinates, g.VertexCoordinate(vertex))
		t.AppendEncodedStep(encodeCoordinate(i, kdtree.MaxEdgeOffset, kdtree.MaxStepOffset))

		edges = g.VertexRawEdges(vertex, edges)
		for j, e := range edges {
			steps := g.EdgeSteps(e, vertex)
			for k, s := range steps {
				coordinates = append(coordinates, s)
				t.AppendEncodedStep(encodeCoordinate(i, j, k))
			}
		}
	}

	createSubTree(t, true)
	return t
}

func subKdTree(t *kdtree.KdTree, from, to int) *kdtree.KdTree {
	// The EncodedSteps slice is restricted by pointers and not with a new slice due to its encoding.
	return &kdtree.KdTree{Graph: t.Graph, EncodedSteps: t.EncodedSteps, Coordinates: t.Coordinates[from:to],
		EncodedStepsStart: t.EncodedStepsStart + from, EncodedStepsEnd: t.EncodedStepsStart + to - 1}
}

func createSubTree(t *kdtree.KdTree, compareLat bool) {
	if t.Len() <= 1 {
		return
	}
	if compareLat {
		sort.Sort(byLat{t})
	} else {
		sort.Sort(byLng{t})
	}
	middle := t.Len() / 2
	createSubTree(subKdTree(t, 0, middle), !compareLat)
	createSubTree(subKdTree(t, middle+1, t.Len()), !compareLat)
}

// writeKdTreeSubgraph creates and stores the k-d tree for the given cluster graph
func writeKdTreeSubgraph(ready chan<- int, baseDir string, g *graph.GraphFile, bboxes []geo.BBox, pos int) {
	t, bbox := createKdTreeSubgraph(g)
	err := writeToFile(t, baseDir)
	if err != nil {
		log.Fatal("Creating k-d tree: ", err)
	}
	// a very simple margin is added
	bbox.Min.Lat -= BBoxMargin
	bbox.Min.Lng -= BBoxMargin
	bbox.Max.Lat += BBoxMargin
	bbox.Max.Lng += BBoxMargin

	bboxes[pos] = bbox
	ready <- 1
}

// writeKdTree creates and stores the k-d tree for the given graph
func writeKdTreeOverlay(baseDir string, g *graph.OverlayGraphFile) {
	t := createKdTreeOverlay(g)
	err := writeToFile(t, baseDir)
	if err != nil {
		log.Fatal("Creating k-d tree: ", err)
	}
}

// writeToFile stores the permitation created by the k-d tree
func writeToFile(t *kdtree.KdTree, baseDir string) error {
	output, err := os.Create(path.Join(baseDir, "kdtree.ftf"))
	defer output.Close()
	if err != nil {
		return err
	}
	writeErr := binary.Write(output, binary.LittleEndian, t.EncodedSteps)
	if writeErr != nil {
		return writeErr
	}
	return nil
}

func encodeCoordinate(vertexIndex, edgeOffset, stepOffset int) uint64 {
	if vertexIndex > kdtree.MaxVertexIndex {
		panic("vertex index too large")
	}
	// both offsets are at max if only a vertex is encoded
	if edgeOffset != kdtree.MaxEdgeOffset && stepOffset != kdtree.MaxStepOffset {
		if edgeOffset >= kdtree.MaxEdgeOffset {
			panic("edge offset too large")
		}
		if stepOffset >= kdtree.MaxStepOffset {
			fmt.Printf("vertex: %d, edge offset: %v, step offset: %v\n", vertexIndex, edgeOffset, stepOffset)
			panic("step offset too large")
		}
	}

	ec := uint64(vertexIndex) << (kdtree.EdgeOffsetBits + kdtree.StepOffsetBits)
	ec |= uint64(edgeOffset) << kdtree.StepOffsetBits
	ec |= uint64(stepOffset)
	return ec
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testing

import (
	"github.com/globocom/tsuru/provision"
	"io"
	"strconv"
)

func init() {
	provision.Register("fake", &FakeProvisioner{})
}

// Fake implementation for provision.Unit.
type FakeUnit struct {
	name    string
	machine int
	actions []string
}

func (u *FakeUnit) GetName() string {
	u.actions = append(u.actions, "getname")
	return u.name
}

func (u *FakeUnit) GetMachine() int {
	u.actions = append(u.actions, "getmachine")
	return u.machine
}

// Fake implementation for provision.App.
type FakeApp struct {
	name      string
	framework string
	units     []provision.AppUnit
	logs      []string
	actions   []string
}

func NewFakeApp(name, framework string, units int) *FakeApp {
	app := FakeApp{
		name:      name,
		framework: framework,
		units:     make([]provision.AppUnit, units),
	}
	for i := 0; i < units; i++ {
		app.units[i] = &FakeUnit{name: name, machine: i + 1}
	}
	return &app
}

func (a *FakeApp) Log(message, source string) error {
	a.logs = append(a.logs, source+message)
	a.actions = append(a.actions, "log "+source+" - "+message)
	return nil
}

func (a *FakeApp) GetName() string {
	a.actions = append(a.actions, "getname")
	return a.name
}

func (a *FakeApp) GetFramework() string {
	a.actions = append(a.actions, "getframework")
	return a.framework
}

func (a *FakeApp) ProvisionUnits() []provision.AppUnit {
	a.actions = append(a.actions, "getunits")
	return a.units
}

type Cmd struct {
	Cmd  string
	Args []string
	App  provision.App
}

// Fake implementation for provision.Provisioner.
type FakeProvisioner struct {
	apps     []provision.App
	Cmds     []Cmd
	outputs  chan []byte
	failures chan error
}

func NewFakeProvisioner() *FakeProvisioner {
	p := FakeProvisioner{}
	p.outputs = make(chan []byte, 8)
	p.failures = make(chan error, 8)
	return &p
}

func (p *FakeProvisioner) FindApp(app provision.App) int {
	for i, a := range p.apps {
		if a.GetName() == app.GetName() {
			return i
		}
	}
	return -1
}

func (p *FakeProvisioner) PrepareOutput(b []byte) {
	p.outputs <- b
}

func (p *FakeProvisioner) PrepareCommandFailure(err error) {
	p.failures <- err
}

func (p *FakeProvisioner) Reset() {
	close(p.outputs)
	close(p.failures)
	p.outputs = make(chan []byte, 8)
	p.failures = make(chan error, 8)
}

func (p *FakeProvisioner) Provision(app provision.App) *provision.Error {
	index := p.FindApp(app)
	if index > -1 {
		return &provision.Error{Reason: "App already provisioned."}
	}
	p.apps = append(p.apps, app)
	return nil
}

func (p *FakeProvisioner) Destroy(app provision.App) *provision.Error {
	index := p.FindApp(app)
	if index == -1 {
		return &provision.Error{Reason: "App is not provisioned."}
	}
	copy(p.apps[index:], p.apps[index+1:])
	p.apps = p.apps[:len(p.apps)-1]
	return nil
}

func (p *FakeProvisioner) ExecuteCommand(w io.Writer, app provision.App, cmd string, args ...string) error {
	var (
		output []byte
		err    error
	)
	command := Cmd{
		Cmd:  cmd,
		Args: args,
		App:  app,
	}
	p.Cmds = append(p.Cmds, command)
	select {
	case output = <-p.outputs:
		w.Write(output)
	case err = <-p.failures:
	}
	return err
}

func (p *FakeProvisioner) CollectStatus() ([]provision.Unit, *provision.Error) {
	units := make([]provision.Unit, len(p.apps))
	for i, app := range p.apps {
		unit := provision.Unit{
			Name:    "somename",
			AppName: app.GetName(),
			Type:    app.GetFramework(),
			Status:  "started",
			Ip:      "10.10.10." + strconv.Itoa(i+1),
			Machine: i + 1,
		}
		units = append(units, unit)
	}
	return units, nil
}
<commit_msg>testing: improve error throwing on FakeProvisioner<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testing

import (
	"github.com/globocom/tsuru/provision"
	"io"
	"strconv"
	"time"
)

func init() {
	provision.Register("fake", &FakeProvisioner{})
}

// Fake implementation for provision.Unit.
type FakeUnit struct {
	name    string
	machine int
	actions []string
}

func (u *FakeUnit) GetName() string {
	u.actions = append(u.actions, "getname")
	return u.name
}

func (u *FakeUnit) GetMachine() int {
	u.actions = append(u.actions, "getmachine")
	return u.machine
}

// Fake implementation for provision.App.
type FakeApp struct {
	name      string
	framework string
	units     []provision.AppUnit
	logs      []string
	actions   []string
}

func NewFakeApp(name, framework string, units int) *FakeApp {
	app := FakeApp{
		name:      name,
		framework: framework,
		units:     make([]provision.AppUnit, units),
	}
	for i := 0; i < units; i++ {
		app.units[i] = &FakeUnit{name: name, machine: i + 1}
	}
	return &app
}

func (a *FakeApp) Log(message, source string) error {
	a.logs = append(a.logs, source+message)
	a.actions = append(a.actions, "log "+source+" - "+message)
	return nil
}

func (a *FakeApp) GetName() string {
	a.actions = append(a.actions, "getname")
	return a.name
}

func (a *FakeApp) GetFramework() string {
	a.actions = append(a.actions, "getframework")
	return a.framework
}

func (a *FakeApp) ProvisionUnits() []provision.AppUnit {
	a.actions = append(a.actions, "getunits")
	return a.units
}

type Cmd struct {
	Cmd  string
	Args []string
	App  provision.App
}

type failure struct {
	method string
	err    error
}

// Fake implementation for provision.Provisioner.
type FakeProvisioner struct {
	apps     []provision.App
	Cmds     []Cmd
	outputs  chan []byte
	failures chan failure
}

func NewFakeProvisioner() *FakeProvisioner {
	p := FakeProvisioner{}
	p.outputs = make(chan []byte, 8)
	p.failures = make(chan failure, 8)
	return &p
}

func (p *FakeProvisioner) FindApp(app provision.App) int {
	for i, a := range p.apps {
		if a.GetName() == app.GetName() {
			return i
		}
	}
	return -1
}

func (p *FakeProvisioner) getError(method string) *provision.Error {
	select {
	case fail := <-p.failures:
		if fail.method == method {
			return &provision.Error{Err: fail.err}
		}
		p.failures <- fail
	case <-time.After(1e9):
	}
	return nil
}

func (p *FakeProvisioner) PrepareOutput(b []byte) {
	p.outputs <- b
}

func (p *FakeProvisioner) PrepareFailure(method string, err error) {
	p.failures <- failure{method, err}
}

func (p *FakeProvisioner) Reset() {
	close(p.outputs)
	close(p.failures)
	p.outputs = make(chan []byte, 8)
	p.failures = make(chan failure, 8)
}

func (p *FakeProvisioner) Provision(app provision.App) *provision.Error {
	if err := p.getError("Provision"); err != nil {
		return err
	}
	index := p.FindApp(app)
	if index > -1 {
		return &provision.Error{Reason: "App already provisioned."}
	}
	p.apps = append(p.apps, app)
	return nil
}

func (p *FakeProvisioner) Destroy(app provision.App) *provision.Error {
	if err := p.getError("Destroy"); err != nil {
		return err
	}
	index := p.FindApp(app)
	if index == -1 {
		return &provision.Error{Reason: "App is not provisioned."}
	}
	copy(p.apps[index:], p.apps[index+1:])
	p.apps = p.apps[:len(p.apps)-1]
	return nil
}

func (p *FakeProvisioner) ExecuteCommand(w io.Writer, app provision.App, cmd string, args ...string) error {
	var (
		output []byte
		err    error
	)
	command := Cmd{
		Cmd:  cmd,
		Args: args,
		App:  app,
	}
	p.Cmds = append(p.Cmds, command)
	select {
	case output = <-p.outputs:
		w.Write(output)
	case fail := <-p.failures:
		if fail.method == "ExecuteCommand" {
			err = fail.err
		} else {
			p.failures <- fail
		}
	}
	return err
}

func (p *FakeProvisioner) CollectStatus() ([]provision.Unit, *provision.Error) {
	if err := p.getError("CollectStatus"); err != nil {
		return nil, err
	}
	units := make([]provision.Unit, len(p.apps))
	for i, app := range p.apps {
		unit := provision.Unit{
			Name:    "somename",
			AppName: app.GetName(),
			Type:    app.GetFramework(),
			Status:  "started",
			Ip:      "10.10.10." + strconv.Itoa(i+1),
			Machine: i + 1,
		}
		units = append(units, unit)
	}
	return units, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Eleme Inc. All rights reserved.

/*

Package config handles configuration parser and container.

Example

Configuration is in JSON file, for example:

	{
	  "interval": 10,
	  "period": [288, 300],
	  "storage": {
	    "path": "storage/"
	  },
	  "detector": {
	    "port": 2015,
	    "factor": 0.05,
	    "blacklist": ["statsd.*"],
	    "intervalHitLimit": 100,
	    "defaultTrustLines": {"timer.count_ps.*": 30},
	    "fillBlankZeros": ["counter.*.exc"]
	  },
	  "webapp": {
	    "port": 2016,
	    "auth": ["user", "pass"],
	    "static": "static/dist"
	  },
	  "alerter": {
	    "command": "",
	    "workers": 4,
	    "interval": 1200,
	    "oneDayLimit": 5
	  }
	}

To use config file with banshee:
	banshee -c path/to/config.json

Documents

The documents for each configuration item with default values:

	interval                   // All metrics incoming interval (in seconds), default: 10
	period                     // All metrics period (in seconds), in form of [NumGrid, GirdLen], default: [288, 300]
	storage.path               // Storage directory path.
	detector.port              // Detector tcp port to listen.
	detector.factor            // Detection weighted moving factor, should be a number between 0 and 1, default: 0.05
	detector.blacklist         // Incoming metrics blacklist, each one should be a wildcard pattern, default: []
	detector.intervalHitLimit  // Limitation for number of filtered metrics for each rule in one interval. default: 100
	detector.defaultTrustLines // Default trustlines for rules, a wildcard pattern to trustline number map. default: {}
	detector.filterBlankZeros  // Detector will fill metric blanks with zeros if it matches any of these wildcard patterns. default: []
	webapp.port                // Webapp http port to listen.
	webapp.auth                // Webapp admin pages basic auth, in form of [user, pass], use empty string ["", ""] to disable auth. default: ["admin", "admin"]
	webapp.static              // Webapp static files (htmls/css/js) path, default: "static/dist"
	alerter.command            // Alerter command or script to execute on anomalies found. default: ""
	alerter.workers            // Number of workers to consume command execution jobs. default: 4
	alerter.interval           // Minimal interval (in seconds) between two alerting message for one metric. default: 1200
	alerter.oneDayLimit        // Limitation for number of alerting times for one metric in a day. default: 5

*/
package config
<commit_msg>Update config for new version config<commit_after>// Copyright 2015 Eleme Inc. All rights reserved.

/*

Package config handles configuration parser and container.

Example

Configuration is in JSON file, for example:

	{
	  "interval": 10,
	  "period": 86400,
	  "storage": {
	    "path": "storage/"
	  },
	  "detector": {
	    "port": 2015,
	    "factor": 0.05,
		"leastCount": 30,
	    "blacklist": ["statsd.*"],
	    "intervalHitLimit": 100,
	    "defaultTrustLines": {"timer.count_ps.*": 30},
	    "fillBlankZeros": ["counter.*.exc"]
	  },
	  "webapp": {
	    "port": 2016,
	    "auth": ["user", "pass"],
	    "static": "static/dist"
	  },
	  "alerter": {
	    "command": "",
	    "workers": 4,
	    "interval": 1200,
	    "oneDayLimit": 5
	  }
	}

To use config file with banshee:
	banshee -c path/to/config.json

Documents

The documents for each configuration item with default values:

	interval                   // All metrics incoming interval (in seconds), default: 10
	period                     // All metrics period (in seconds), default: 86400 (1 day).
	storage.path               // Storage directory path.
	detector.port              // Detector tcp port to listen.
	detector.factor            // Detection weighted moving factor, should be a number between 0 and 1, default: 0.05
	detector.leastCount        // Least count to start detection. default: 30
	detector.blacklist         // Incoming metrics blacklist, each one should be a wildcard pattern, default: []
	detector.intervalHitLimit  // Limitation for number of filtered metrics for each rule in one interval. default: 100
	detector.defaultTrustLines // Default trustlines for rules, a wildcard pattern to trustline number map. default: {}
	detector.filterBlankZeros  // Detector will fill metric blanks with zeros if it matches any of these wildcard patterns. default: []
	webapp.port                // Webapp http port to listen.
	webapp.auth                // Webapp admin pages basic auth, in form of [user, pass], use empty string ["", ""] to disable auth. default: ["admin", "admin"]
	webapp.static              // Webapp static files (htmls/css/js) path, default: "static/dist"
	alerter.command            // Alerter command or script to execute on anomalies found. default: ""
	alerter.workers            // Number of workers to consume command execution jobs. default: 4
	alerter.interval           // Minimal interval (in seconds) between two alerting message for one metric. default: 1200
	alerter.oneDayLimit        // Limitation for number of alerting times for one metric in a day. default: 5

*/
package config
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
)

const ProxyAuthenticationRequired = "HTTP/1.0 407 Proxy authentication required\r\n\r\n"

type connection struct {
	id       string
	incoming net.Conn
	outgoing net.Conn
	proxy
	localAddr net.Addr
}

func (c *connection) Dial(network, address string) (net.Conn, error) {
	if c.localAddr == nil {
		logger.Warn.Println(c.id, "Missing local net.Addr: a default local net.Addr will be used")
		goto fallback
	}

	if config.UseIncomingLocalAddr {
		// Ensure the TCPAddr has its Port set to 0, which is way of telling the dialer to use
		// and random port.
		switch tcpAddr := c.localAddr.(type) {
		case *net.TCPAddr:
			tcpAddr.Port = 0
		default:
			logger.Warn.Println(c.id, "Ignoring local net.Addr", c.localAddr, "because net.TCPAddr was expected")
			goto fallback
		}

		dialer := &net.Dialer{LocalAddr: c.localAddr}

		// Try to dial with the incoming LocalAddr to keep the incoming and outgoing IPs the same.
		conn, err := dialer.Dial(network, address)
		if err == nil {
			return conn, nil
		}

		// If an error occurs, fallback to the default interface. This might happen if you connected
		// via a loopback interace, like testing on the same machine. We should be more specifc about
		// error handling, but falling back is fine for now.
		logger.Warn.Println(c.id, "Ignoring local net.Addr for", c.localAddr, "dialing due to error:", err)
	}

fallback:
	return net.Dial(network, address)
}

func (c *connection) Handle() {
	logger.Info.Println(c.id, "Handling new connection.")

	reader := bufio.NewReader(c.incoming)
	request, err := http.ReadRequest(reader)
	if err == io.EOF {
		logger.Warn.Println(c.id, "Incoming connection disconnected.")
		return
	}
	if err != nil {
		logger.Warn.Println(c.id, "Could not parse or read request from incoming connection:", err)
		return
	}

	defer request.Body.Close()

	if !isAuthenticated(request) {
		logger.Fatal.Println(c.id, "Invalid credentials.")
		c.incoming.Write([]byte(ProxyAuthenticationRequired))
		return
	}

	// Delete the auth and proxy headers.
	if config.AuthenticationRequired() {
		request.Header.Del("Proxy-Authorization")
	}

	// Delete any other proxy related thing if enabled.
	if config.StripProxyHeaders {
		request.Header.Del("Forwarded")
		request.Header.Del("Proxy-Connection")
		request.Header.Del("Via")
		request.Header.Del("X-Forwarded-For")
		request.Header.Del("X-Forwarded-Host")
		request.Header.Del("X-Forwarded-Proto")
	}

	logger.Info.Println(c.id, "Processing connection to:", request.Method, request.Host)

	if request.Method == "CONNECT" {
		c.proxy = &httpsProxy{}
	} else {
		c.proxy = &httpProxy{}
	}

	err = c.proxy.SetupOutgoing(c, request)
	if err != nil {
		logger.Warn.Println(c.id, err)
		return
	}

	// Spawn incoming->outgoing and outgoing->incoming streams.
	signal := make(chan error, 1)
	go streamBytes(c.incoming, c.outgoing, signal)
	go streamBytes(c.outgoing, c.incoming, signal)

	// Wait for either stream to complete and finish. The second will always be an error.
	err = <-signal
	if err != nil {
		logger.Warn.Println(c.id, "Error reading or writing data", request.Host, err)
		return
	}
}

func (c *connection) Close() {
	if c.incoming != nil {
		c.incoming.Close()
	}

	if c.outgoing != nil {
		c.outgoing.Close()
	}

	logger.Info.Println(c.id, "Connection closed.")
}

// COPIED FROM STD LIB TO USE WITH PROXY-AUTH HEADER
// parseBasicAuth parses an HTTP Basic Authentication string.
// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true).
func parseBasicAuth(auth string) (username, password string, ok bool) {
	const prefix = "Basic "
	if !strings.HasPrefix(auth, prefix) {
		return
	}
	c, err := base64.StdEncoding.DecodeString(auth[len(prefix):])
	if err != nil {
		return
	}
	cs := string(c)
	s := strings.IndexByte(cs, ':')
	if s < 0 {
		return
	}
	return cs[:s], cs[s+1:], true
}

func isAuthenticated(request *http.Request) bool {
	if !config.AuthenticationRequired() {
		return true
	}

	proxyAuthHeader := request.Header.Get("Proxy-Authorization")
	if proxyAuthHeader == "" {
		return false
	}

	username, password, ok := parseBasicAuth(proxyAuthHeader)
	if !ok {
		return false
	}

	return config.IsAuthenticated(username, password)
}

func newConnectionId() string {
	bytes := make([]byte, 3) // 6 characters long.
	if _, err := rand.Read(bytes); err != nil {
		return "[ERROR-MAKING-ID]"
	}
	return "[" + hex.EncodeToString(bytes) + "]"
}

func NewConnection(incoming net.Conn) *connection {
	newId := fmt.Sprint(newConnectionId(), " [", incoming.RemoteAddr().String(), "]")
	localAddr := incoming.LocalAddr()

	return &connection{
		id:        newId,
		incoming:  incoming,
		localAddr: localAddr,
	}
}
<commit_msg>Move localAddr check to inside the config conditional<commit_after>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
)

const ProxyAuthenticationRequired = "HTTP/1.0 407 Proxy authentication required\r\n\r\n"

type connection struct {
	id       string
	incoming net.Conn
	outgoing net.Conn
	proxy
	localAddr net.Addr
}

func (c *connection) Dial(network, address string) (net.Conn, error) {
	if config.UseIncomingLocalAddr {
		if c.localAddr == nil {
			logger.Warn.Println(c.id, "Missing local net.Addr: a default local net.Addr will be used")
			goto fallback
		}

		// Ensure the TCPAddr has its Port set to 0, which is way of telling the dialer to use
		// and random port.
		switch tcpAddr := c.localAddr.(type) {
		case *net.TCPAddr:
			tcpAddr.Port = 0
		default:
			logger.Warn.Println(c.id, "Ignoring local net.Addr", c.localAddr, "because net.TCPAddr was expected")
			goto fallback
		}

		dialer := &net.Dialer{LocalAddr: c.localAddr}

		// Try to dial with the incoming LocalAddr to keep the incoming and outgoing IPs the same.
		conn, err := dialer.Dial(network, address)
		if err == nil {
			return conn, nil
		}

		// If an error occurs, fallback to the default interface. This might happen if you connected
		// via a loopback interace, like testing on the same machine. We should be more specifc about
		// error handling, but falling back is fine for now.
		logger.Warn.Println(c.id, "Ignoring local net.Addr for", c.localAddr, "dialing due to error:", err)
	}

fallback:
	return net.Dial(network, address)
}

func (c *connection) Handle() {
	logger.Info.Println(c.id, "Handling new connection.")

	reader := bufio.NewReader(c.incoming)
	request, err := http.ReadRequest(reader)
	if err == io.EOF {
		logger.Warn.Println(c.id, "Incoming connection disconnected.")
		return
	}
	if err != nil {
		logger.Warn.Println(c.id, "Could not parse or read request from incoming connection:", err)
		return
	}

	defer request.Body.Close()

	if !isAuthenticated(request) {
		logger.Fatal.Println(c.id, "Invalid credentials.")
		c.incoming.Write([]byte(ProxyAuthenticationRequired))
		return
	}

	// Delete the auth and proxy headers.
	if config.AuthenticationRequired() {
		request.Header.Del("Proxy-Authorization")
	}

	// Delete any other proxy related thing if enabled.
	if config.StripProxyHeaders {
		request.Header.Del("Forwarded")
		request.Header.Del("Proxy-Connection")
		request.Header.Del("Via")
		request.Header.Del("X-Forwarded-For")
		request.Header.Del("X-Forwarded-Host")
		request.Header.Del("X-Forwarded-Proto")
	}

	logger.Info.Println(c.id, "Processing connection to:", request.Method, request.Host)

	if request.Method == "CONNECT" {
		c.proxy = &httpsProxy{}
	} else {
		c.proxy = &httpProxy{}
	}

	err = c.proxy.SetupOutgoing(c, request)
	if err != nil {
		logger.Warn.Println(c.id, err)
		return
	}

	// Spawn incoming->outgoing and outgoing->incoming streams.
	signal := make(chan error, 1)
	go streamBytes(c.incoming, c.outgoing, signal)
	go streamBytes(c.outgoing, c.incoming, signal)

	// Wait for either stream to complete and finish. The second will always be an error.
	err = <-signal
	if err != nil {
		logger.Warn.Println(c.id, "Error reading or writing data", request.Host, err)
		return
	}
}

func (c *connection) Close() {
	if c.incoming != nil {
		c.incoming.Close()
	}

	if c.outgoing != nil {
		c.outgoing.Close()
	}

	logger.Info.Println(c.id, "Connection closed.")
}

// COPIED FROM STD LIB TO USE WITH PROXY-AUTH HEADER
// parseBasicAuth parses an HTTP Basic Authentication string.
// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true).
func parseBasicAuth(auth string) (username, password string, ok bool) {
	const prefix = "Basic "
	if !strings.HasPrefix(auth, prefix) {
		return
	}
	c, err := base64.StdEncoding.DecodeString(auth[len(prefix):])
	if err != nil {
		return
	}
	cs := string(c)
	s := strings.IndexByte(cs, ':')
	if s < 0 {
		return
	}
	return cs[:s], cs[s+1:], true
}

func isAuthenticated(request *http.Request) bool {
	if !config.AuthenticationRequired() {
		return true
	}

	proxyAuthHeader := request.Header.Get("Proxy-Authorization")
	if proxyAuthHeader == "" {
		return false
	}

	username, password, ok := parseBasicAuth(proxyAuthHeader)
	if !ok {
		return false
	}

	return config.IsAuthenticated(username, password)
}

func newConnectionId() string {
	bytes := make([]byte, 3) // 6 characters long.
	if _, err := rand.Read(bytes); err != nil {
		return "[ERROR-MAKING-ID]"
	}
	return "[" + hex.EncodeToString(bytes) + "]"
}

func NewConnection(incoming net.Conn) *connection {
	newId := fmt.Sprint(newConnectionId(), " [", incoming.RemoteAddr().String(), "]")
	localAddr := incoming.LocalAddr()

	return &connection{
		id:        newId,
		incoming:  incoming,
		localAddr: localAddr,
	}
}
<|endoftext|>


<commit_before>package gremgo

import (
	"net/http"
	"time"

	"github.com/gorilla/websocket"
	"sync"
)

type dialer interface {
	connect() error
	isConnected() bool
	isDisposed() bool
	write([]byte) error
	read() ([]byte, error)
	close() error
	getAuth() *auth
	ping(errs chan error)
}

/////
/*
WebSocket Connection
*/
/////

// Ws is the dialer for a WebSocket connection
type Ws struct {
	host         string
	conn         *websocket.Conn
	auth         *auth
	disposed     bool
	connected    bool
	pingInterval time.Duration
	writingWait  time.Duration
	readingWait  time.Duration
	timeout      time.Duration
	quit         chan struct{}
	sync.RWMutex
}

//Auth is the container for authentication data of dialer
type auth struct {
	username string
	password string
}

func (ws *Ws) connect() (err error) {
	d := websocket.Dialer{
		WriteBufferSize:  524288,
		ReadBufferSize:   524288,
		HandshakeTimeout: 5 * time.Second, // Timeout or else we'll hang forever and never fail on bad hosts.
	}
	ws.conn, _, err = d.Dial(ws.host, http.Header{})
	if err != nil {

		// As of 3.2.2 the URL has changed.
		// https://groups.google.com/forum/#!msg/gremlin-users/x4hiHsmTsHM/Xe4GcPtRCAAJ
		ws.host = ws.host + "/gremlin"
		ws.conn, _, err = d.Dial(ws.host, http.Header{})
	}

	if err == nil {
		ws.connected = true
		ws.conn.SetPongHandler(func(appData string) error {
			ws.Lock()
			ws.connected = true
			ws.Unlock()
			return nil
		})
	}
	return
}

func (ws *Ws) isConnected() bool {
	return ws.connected
}

func (ws *Ws) isDisposed() bool {
	return ws.disposed
}

func (ws *Ws) write(msg []byte) (err error) {
	err = ws.conn.WriteMessage(2, msg)
	return
}

func (ws *Ws) read() (msg []byte, err error) {
	_, msg, err = ws.conn.ReadMessage()
	return
}

func (ws *Ws) close() (err error) {
	defer func() {
		close(ws.quit)
		ws.conn.Close()
		ws.disposed = true
	}()

	err = ws.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) //Cleanly close the connection with the server
	return
}

func (ws *Ws) getAuth() *auth {
	if ws.auth == nil {
		panic("You must create a Secure Dialer for authenticate with the server")
	}
	return ws.auth
}

func (ws *Ws) ping(errs chan error) {
	ticker := time.NewTicker(ws.pingInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			connected := true
			if err := ws.conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(ws.writingWait)); err != nil {
				errs <- err
				connected = false
			}
			ws.Lock()
			ws.connected = connected
			ws.Unlock()

		case <-ws.quit:
			return
		}
	}
}

/////

func (c *Client) writeWorker(errs chan error, quit chan struct{}) { // writeWorker works on a loop and dispatches messages as soon as it recieves them
	for {
		select {
		case msg := <-c.requests:
			err := c.conn.write(msg)
			if err != nil {
				errs <- err
				c.Errored = true
				break
			}

		case <-quit:
			return
		}
	}
}

func (c *Client) readWorker(errs chan error, quit chan struct{}) { // readWorker works on a loop and sorts messages as soon as it recieves them
	for {
		msg, err := c.conn.read()
		if err != nil {
			errs <- err
			c.Errored = true
			break
		}
		if msg != nil {
			err = c.handleResponse(msg)
		}
		if err != nil {
			errs <- err
			c.Errored = true
			break
		}

		select {
		case <-quit:
			return
		default:
			continue
		}
	}
}
<commit_msg>dont send error to channel, that will make client thinks conn error<commit_after>package gremgo

import (
	"net/http"
	"time"

	"github.com/gorilla/websocket"
	"sync"
)

type dialer interface {
	connect() error
	isConnected() bool
	isDisposed() bool
	write([]byte) error
	read() ([]byte, error)
	close() error
	getAuth() *auth
	ping(errs chan error)
}

/////
/*
WebSocket Connection
*/
/////

// Ws is the dialer for a WebSocket connection
type Ws struct {
	host         string
	conn         *websocket.Conn
	auth         *auth
	disposed     bool
	connected    bool
	pingInterval time.Duration
	writingWait  time.Duration
	readingWait  time.Duration
	timeout      time.Duration
	quit         chan struct{}
	sync.RWMutex
}

//Auth is the container for authentication data of dialer
type auth struct {
	username string
	password string
}

func (ws *Ws) connect() (err error) {
	d := websocket.Dialer{
		WriteBufferSize:  524288,
		ReadBufferSize:   524288,
		HandshakeTimeout: 5 * time.Second, // Timeout or else we'll hang forever and never fail on bad hosts.
	}
	ws.conn, _, err = d.Dial(ws.host, http.Header{})
	if err != nil {

		// As of 3.2.2 the URL has changed.
		// https://groups.google.com/forum/#!msg/gremlin-users/x4hiHsmTsHM/Xe4GcPtRCAAJ
		ws.host = ws.host + "/gremlin"
		ws.conn, _, err = d.Dial(ws.host, http.Header{})
	}

	if err == nil {
		ws.connected = true
		ws.conn.SetPongHandler(func(appData string) error {
			ws.Lock()
			ws.connected = true
			ws.Unlock()
			return nil
		})
	}
	return
}

func (ws *Ws) isConnected() bool {
	return ws.connected
}

func (ws *Ws) isDisposed() bool {
	return ws.disposed
}

func (ws *Ws) write(msg []byte) (err error) {
	err = ws.conn.WriteMessage(2, msg)
	return
}

func (ws *Ws) read() (msg []byte, err error) {
	_, msg, err = ws.conn.ReadMessage()
	return
}

func (ws *Ws) close() (err error) {
	defer func() {
		close(ws.quit)
		ws.conn.Close()
		ws.disposed = true
	}()

	err = ws.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) //Cleanly close the connection with the server
	return
}

func (ws *Ws) getAuth() *auth {
	if ws.auth == nil {
		panic("You must create a Secure Dialer for authenticate with the server")
	}
	return ws.auth
}

func (ws *Ws) ping(errs chan error) {
	ticker := time.NewTicker(ws.pingInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			connected := true
			if err := ws.conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(ws.writingWait)); err != nil {
				errs <- err
				connected = false
			}
			ws.Lock()
			ws.connected = connected
			ws.Unlock()

		case <-ws.quit:
			return
		}
	}
}

/////

func (c *Client) writeWorker(errs chan error, quit chan struct{}) { // writeWorker works on a loop and dispatches messages as soon as it recieves them
	for {
		select {
		case msg := <-c.requests:
			err := c.conn.write(msg)
			if err != nil {
				errs <- err
				c.Errored = true
				break
			}

		case <-quit:
			return
		}
	}
}

func (c *Client) readWorker(errs chan error, quit chan struct{}) { // readWorker works on a loop and sorts messages as soon as it recieves them
	for {
		msg, err := c.conn.read()
		if err != nil {
			errs <- err
			c.Errored = true
			break
		}
		if msg != nil {
			err = c.handleResponse(msg)
		}
		if err != nil {
			c.Errored = true
			break
		}

		select {
		case <-quit:
			return
		default:
			continue
		}
	}
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/raft"
	"github.com/ugorji/go/codec"
	"io"
	"log"
	"time"
)

// consulFSM implements a finite state machine that is used
// along with Raft to provide strong consistency. We implement
// this outside the Server to avoid exposing this outside the package.
type consulFSM struct {
	state *StateStore
}

// consulSnapshot is used to provide a snapshot of the current
// state in a way that can be accessed concurrently with operations
// that may modify the live state.
type consulSnapshot struct {
	state *StateSnapshot
}

// NewFSM is used to construct a new FSM with a blank state
func NewFSM() (*consulFSM, error) {
	state, err := NewStateStore()
	if err != nil {
		return nil, err
	}

	fsm := &consulFSM{
		state: state,
	}
	return fsm, nil
}

// State is used to return a handle to the current state
func (c *consulFSM) State() *StateStore {
	return c.state
}

func (c *consulFSM) Apply(buf []byte) interface{} {
	switch structs.MessageType(buf[0]) {
	case structs.RegisterRequestType:
		return c.applyRegister(buf[1:])
	case structs.DeregisterRequestType:
		return c.applyDeregister(buf[1:])
	default:
		panic(fmt.Errorf("failed to apply request: %#v", buf))
	}
}

func (c *consulFSM) applyRegister(buf []byte) interface{} {
	var req structs.RegisterRequest
	if err := structs.Decode(buf, &req); err != nil {
		panic(fmt.Errorf("failed to decode request: %v", err))
	}

	// Ensure the node
	c.state.EnsureNode(req.Node, req.Address)

	// Ensure the service if provided
	if req.ServiceName != "" {
		c.state.EnsureService(req.Node, req.ServiceName, req.ServiceTag, req.ServicePort)
	}
	return nil
}

func (c *consulFSM) applyDeregister(buf []byte) interface{} {
	var req structs.DeregisterRequest
	if err := structs.Decode(buf, &req); err != nil {
		panic(fmt.Errorf("failed to decode request: %v", err))
	}

	// Either remove the service entry or the whole node
	if req.ServiceName != "" {
		c.state.DeleteNodeService(req.Node, req.ServiceName)
	} else {
		c.state.DeleteNode(req.Node)
	}
	return nil
}

func (c *consulFSM) Snapshot() (raft.FSMSnapshot, error) {
	defer func(start time.Time) {
		log.Printf("[INFO] FSM Snapshot created in %v", time.Now().Sub(start))
	}(time.Now())

	// Create a new snapshot
	snap, err := c.state.Snapshot()
	if err != nil {
		return nil, err
	}
	return &consulSnapshot{snap}, nil
}

func (c *consulFSM) Restore(old io.ReadCloser) error {
	defer old.Close()

	// Create a new state store
	state, err := NewStateStore()
	if err != nil {
		return err
	}

	// Create a decoder
	var handle codec.MsgpackHandle
	dec := codec.NewDecoder(old, &handle)

	// Populate the new state
	msgType := make([]byte, 1)
	for {
		// Read the message type
		_, err := old.Read(msgType)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		// Decode
		switch structs.MessageType(msgType[0]) {
		case structs.RegisterRequestType:
			var req structs.RegisterRequest
			if err := dec.Decode(&req); err != nil {
				return err
			}

			// Register the service or the node
			if req.ServiceName != "" {
				state.EnsureService(req.Node, req.ServiceName,
					req.ServiceTag, req.ServicePort)
			} else {
				state.EnsureNode(req.Node, req.Address)
			}

		default:
			return fmt.Errorf("Unrecognized msg type: %v", msgType)
		}
	}

	// Do an atomic flip, safe since Apply is not called concurrently
	c.state = state
	return nil
}

func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error {
	// Get all the nodes
	nodes := s.state.Nodes()

	// Register the nodes
	handle := codec.MsgpackHandle{}
	encoder := codec.NewEncoder(sink, &handle)

	// Register each node
	var req structs.RegisterRequest
	for i := 0; i < len(nodes); i += 2 {
		req = structs.RegisterRequest{
			Node:    nodes[i],
			Address: nodes[i+1],
		}

		// Register the node itself
		sink.Write([]byte{byte(structs.RegisterRequestType)})
		if err := encoder.Encode(&req); err != nil {
			sink.Cancel()
			return err
		}

		// Register each service this node has
		services := s.state.NodeServices(nodes[i])
		for serv, props := range services.Services {
			req.ServiceName = serv
			req.ServiceTag = props.Tag
			req.ServicePort = props.Port

			sink.Write([]byte{byte(structs.RegisterRequestType)})
			if err := encoder.Encode(&req); err != nil {
				sink.Cancel()
				return err
			}
		}
	}
	return nil
}

func (s *consulSnapshot) Release() {
	s.state.Close()
}
<commit_msg>Handle new Raft API<commit_after>package consul

import (
	"fmt"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/raft"
	"github.com/ugorji/go/codec"
	"io"
	"log"
	"time"
)

// consulFSM implements a finite state machine that is used
// along with Raft to provide strong consistency. We implement
// this outside the Server to avoid exposing this outside the package.
type consulFSM struct {
	state *StateStore
}

// consulSnapshot is used to provide a snapshot of the current
// state in a way that can be accessed concurrently with operations
// that may modify the live state.
type consulSnapshot struct {
	state *StateSnapshot
}

// NewFSM is used to construct a new FSM with a blank state
func NewFSM() (*consulFSM, error) {
	state, err := NewStateStore()
	if err != nil {
		return nil, err
	}

	fsm := &consulFSM{
		state: state,
	}
	return fsm, nil
}

// State is used to return a handle to the current state
func (c *consulFSM) State() *StateStore {
	return c.state
}

func (c *consulFSM) Apply(log *raft.Log) interface{} {
	buf := log.Data
	switch structs.MessageType(buf[0]) {
	case structs.RegisterRequestType:
		return c.applyRegister(buf[1:])
	case structs.DeregisterRequestType:
		return c.applyDeregister(buf[1:])
	default:
		panic(fmt.Errorf("failed to apply request: %#v", buf))
	}
}

func (c *consulFSM) applyRegister(buf []byte) interface{} {
	var req structs.RegisterRequest
	if err := structs.Decode(buf, &req); err != nil {
		panic(fmt.Errorf("failed to decode request: %v", err))
	}

	// Ensure the node
	c.state.EnsureNode(req.Node, req.Address)

	// Ensure the service if provided
	if req.ServiceName != "" {
		c.state.EnsureService(req.Node, req.ServiceName, req.ServiceTag, req.ServicePort)
	}
	return nil
}

func (c *consulFSM) applyDeregister(buf []byte) interface{} {
	var req structs.DeregisterRequest
	if err := structs.Decode(buf, &req); err != nil {
		panic(fmt.Errorf("failed to decode request: %v", err))
	}

	// Either remove the service entry or the whole node
	if req.ServiceName != "" {
		c.state.DeleteNodeService(req.Node, req.ServiceName)
	} else {
		c.state.DeleteNode(req.Node)
	}
	return nil
}

func (c *consulFSM) Snapshot() (raft.FSMSnapshot, error) {
	defer func(start time.Time) {
		log.Printf("[INFO] FSM Snapshot created in %v", time.Now().Sub(start))
	}(time.Now())

	// Create a new snapshot
	snap, err := c.state.Snapshot()
	if err != nil {
		return nil, err
	}
	return &consulSnapshot{snap}, nil
}

func (c *consulFSM) Restore(old io.ReadCloser) error {
	defer old.Close()

	// Create a new state store
	state, err := NewStateStore()
	if err != nil {
		return err
	}

	// Create a decoder
	var handle codec.MsgpackHandle
	dec := codec.NewDecoder(old, &handle)

	// Populate the new state
	msgType := make([]byte, 1)
	for {
		// Read the message type
		_, err := old.Read(msgType)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		// Decode
		switch structs.MessageType(msgType[0]) {
		case structs.RegisterRequestType:
			var req structs.RegisterRequest
			if err := dec.Decode(&req); err != nil {
				return err
			}

			// Register the service or the node
			if req.ServiceName != "" {
				state.EnsureService(req.Node, req.ServiceName,
					req.ServiceTag, req.ServicePort)
			} else {
				state.EnsureNode(req.Node, req.Address)
			}

		default:
			return fmt.Errorf("Unrecognized msg type: %v", msgType)
		}
	}

	// Do an atomic flip, safe since Apply is not called concurrently
	c.state = state
	return nil
}

func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error {
	// Get all the nodes
	nodes := s.state.Nodes()

	// Register the nodes
	handle := codec.MsgpackHandle{}
	encoder := codec.NewEncoder(sink, &handle)

	// Register each node
	var req structs.RegisterRequest
	for i := 0; i < len(nodes); i += 2 {
		req = structs.RegisterRequest{
			Node:    nodes[i],
			Address: nodes[i+1],
		}

		// Register the node itself
		sink.Write([]byte{byte(structs.RegisterRequestType)})
		if err := encoder.Encode(&req); err != nil {
			sink.Cancel()
			return err
		}

		// Register each service this node has
		services := s.state.NodeServices(nodes[i])
		for serv, props := range services.Services {
			req.ServiceName = serv
			req.ServiceTag = props.Tag
			req.ServicePort = props.Port

			sink.Write([]byte{byte(structs.RegisterRequestType)})
			if err := encoder.Encode(&req); err != nil {
				sink.Cancel()
				return err
			}
		}
	}
	return nil
}

func (s *consulSnapshot) Release() {
	s.state.Close()
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"os"
	"runtime"

	"github.com/op/go-logging"
	"github.com/spf13/viper"
	"golang.org/x/net/context"

	"github.com/golang/protobuf/ptypes/empty"
	"github.com/hyperledger/fabric/common/flogging"
	pb "github.com/hyperledger/fabric/protos/peer"
)

var log = logging.MustGetLogger("server")

// NewAdminServer creates and returns a Admin service instance.
func NewAdminServer() *ServerAdmin {
	s := new(ServerAdmin)
	return s
}

// ServerAdmin implementation of the Admin service for the Peer
type ServerAdmin struct {
}

func worker(id int, die chan struct{}) {
	for {
		select {
		case <-die:
			log.Debugf("worker %d terminating", id)
			return
		default:
			log.Debugf("%d is working...", id)
			runtime.Gosched()
		}
	}
}

// GetStatus reports the status of the server
func (*ServerAdmin) GetStatus(context.Context, *empty.Empty) (*pb.ServerStatus, error) {
	status := &pb.ServerStatus{Status: pb.ServerStatus_STARTED}
	log.Debugf("returning status: %s", status)
	return status, nil
}

// StartServer starts the server
func (*ServerAdmin) StartServer(context.Context, *empty.Empty) (*pb.ServerStatus, error) {
	status := &pb.ServerStatus{Status: pb.ServerStatus_STARTED}
	log.Debugf("returning status: %s", status)
	return status, nil
}

// StopServer stops the server
func (*ServerAdmin) StopServer(context.Context, *empty.Empty) (*pb.ServerStatus, error) {
	status := &pb.ServerStatus{Status: pb.ServerStatus_STOPPED}
	log.Debugf("returning status: %s", status)

	pidFile := viper.GetString("peer.fileSystemPath") + "/peer.pid"
	log.Debugf("Remove pid file  %s", pidFile)
	os.Remove(pidFile)
	defer os.Exit(0)
	return status, nil
}

// GetModuleLogLevel gets the current logging level for the specified module
// TODO Modify the signature so as to remove the error return - it's always been nil
func (*ServerAdmin) GetModuleLogLevel(ctx context.Context, request *pb.LogLevelRequest) (*pb.LogLevelResponse, error) {
	logLevelString := flogging.GetModuleLevel(request.LogModule)
	logResponse := &pb.LogLevelResponse{LogModule: request.LogModule, LogLevel: logLevelString}
	return logResponse, nil
}

// SetModuleLogLevel sets the logging level for the specified module
func (*ServerAdmin) SetModuleLogLevel(ctx context.Context, request *pb.LogLevelRequest) (*pb.LogLevelResponse, error) {
	logLevelString, err := flogging.SetModuleLevel(request.LogModule, request.LogLevel)
	logResponse := &pb.LogLevelResponse{LogModule: request.LogModule, LogLevel: logLevelString}
	return logResponse, err
}

// RevertLogLevels reverts the log levels for all modules to the level
// defined at the end of peer startup.
func (*ServerAdmin) RevertLogLevels(context.Context, *empty.Empty) (*empty.Empty, error) {
	err := flogging.RevertToPeerStartupLevels()

	return &empty.Empty{}, err
}
<commit_msg>FAB-3187 remove dead code in core/admin.go<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"os"

	"github.com/op/go-logging"
	"github.com/spf13/viper"
	"golang.org/x/net/context"

	"github.com/golang/protobuf/ptypes/empty"
	"github.com/hyperledger/fabric/common/flogging"
	pb "github.com/hyperledger/fabric/protos/peer"
)

var log = logging.MustGetLogger("server")

// NewAdminServer creates and returns a Admin service instance.
func NewAdminServer() *ServerAdmin {
	s := new(ServerAdmin)
	return s
}

// ServerAdmin implementation of the Admin service for the Peer
type ServerAdmin struct {
}

// GetStatus reports the status of the server
func (*ServerAdmin) GetStatus(context.Context, *empty.Empty) (*pb.ServerStatus, error) {
	status := &pb.ServerStatus{Status: pb.ServerStatus_STARTED}
	log.Debugf("returning status: %s", status)
	return status, nil
}

// StartServer starts the server
func (*ServerAdmin) StartServer(context.Context, *empty.Empty) (*pb.ServerStatus, error) {
	status := &pb.ServerStatus{Status: pb.ServerStatus_STARTED}
	log.Debugf("returning status: %s", status)
	return status, nil
}

// StopServer stops the server
func (*ServerAdmin) StopServer(context.Context, *empty.Empty) (*pb.ServerStatus, error) {
	status := &pb.ServerStatus{Status: pb.ServerStatus_STOPPED}
	log.Debugf("returning status: %s", status)

	pidFile := viper.GetString("peer.fileSystemPath") + "/peer.pid"
	log.Debugf("Remove pid file  %s", pidFile)
	os.Remove(pidFile)
	defer os.Exit(0)
	return status, nil
}

// GetModuleLogLevel gets the current logging level for the specified module
// TODO Modify the signature so as to remove the error return - it's always been nil
func (*ServerAdmin) GetModuleLogLevel(ctx context.Context, request *pb.LogLevelRequest) (*pb.LogLevelResponse, error) {
	logLevelString := flogging.GetModuleLevel(request.LogModule)
	logResponse := &pb.LogLevelResponse{LogModule: request.LogModule, LogLevel: logLevelString}
	return logResponse, nil
}

// SetModuleLogLevel sets the logging level for the specified module
func (*ServerAdmin) SetModuleLogLevel(ctx context.Context, request *pb.LogLevelRequest) (*pb.LogLevelResponse, error) {
	logLevelString, err := flogging.SetModuleLevel(request.LogModule, request.LogLevel)
	logResponse := &pb.LogLevelResponse{LogModule: request.LogModule, LogLevel: logLevelString}
	return logResponse, err
}

// RevertLogLevels reverts the log levels for all modules to the level
// defined at the end of peer startup.
func (*ServerAdmin) RevertLogLevels(context.Context, *empty.Empty) (*empty.Empty, error) {
	err := flogging.RevertToPeerStartupLevels()

	return &empty.Empty{}, err
}
<|endoftext|>


<commit_before>// Copyright 2014 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author：polaris	studygolang@gmail.com

package service

import (
	"errors"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"logger"
	"model"
	"util"

	"github.com/PuerkitoBio/goquery"
)

var domainPatch = map[string]string{
	"iteye.com":      "iteye.com",
	"blog.51cto.com": "blog.51cto.com",
}

var articleRe = regexp.MustCompile("[\r　\n  \t\v]+")
var articleSpaceRe = regexp.MustCompile("[ ]+")

// 获取url对应的文章并根据规则进行解析
func ParseArticle(articleUrl string, auto bool) (*model.Article, error) {
	articleUrl = strings.TrimSpace(articleUrl)
	if !strings.HasPrefix(articleUrl, "http") {
		articleUrl = "http://" + articleUrl
	}

	tmpArticle := model.NewArticle()
	err := tmpArticle.Where("url=" + articleUrl).Find("id")
	if err != nil || tmpArticle.Id != 0 {
		logger.Errorln(articleUrl, "has exists:", err)
		return nil, errors.New("has exists!")
	}

	urlPaths := strings.SplitN(articleUrl, "/", 5)
	domain := urlPaths[2]

	for k, v := range domainPatch {
		if strings.Contains(domain, k) && !strings.Contains(domain, "www."+k) {
			domain = v
			break
		}
	}

	rule := model.NewCrawlRule()
	err = rule.Where("domain=" + domain).Find()
	if err != nil {
		logger.Errorln("find rule by domain error:", err)
		return nil, err
	}

	if rule.Id == 0 {
		logger.Errorln("domain:", domain, "not exists!")
		return nil, errors.New("domain not exists")
	}

	var doc *goquery.Document
	if doc, err = goquery.NewDocument(articleUrl); err != nil {
		logger.Errorln("goquery newdocument error:", err)
		return nil, err
	}

	author, authorTxt := "", ""
	if rule.InUrl {
		index, err := strconv.Atoi(rule.Author)
		if err != nil {
			logger.Errorln("author rule is illegal:", rule.Author, "error:", err)
			return nil, err
		}
		author = urlPaths[index]
		authorTxt = author
	} else {
		if strings.HasPrefix(rule.Author, ".") || strings.HasPrefix(rule.Author, "#") {
			authorSelection := doc.Find(rule.Author)
			author, err = authorSelection.Html()
			if err != nil {
				logger.Errorln("goquery parse author error:", err)
				return nil, err
			}

			author = strings.TrimSpace(author)
			authorTxt = strings.TrimSpace(authorSelection.Text())
		} else {
			// 某些个人博客，页面中没有作者的信息，因此，规则中 author 即为 作者
			author = rule.Author
			authorTxt = rule.Author
		}
	}

	title := ""
	doc.Find(rule.Title).Each(func(i int, selection *goquery.Selection) {
		if title != "" {
			return
		}

		tmpTitle := strings.TrimSpace(strings.TrimPrefix(selection.Text(), "原"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "荐"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "转"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "顶"))
		if tmpTitle != "" {
			title = tmpTitle
		}
	})

	if title == "" {
		logger.Errorln("url:", articleUrl, "parse title error:", err)
		return nil, err
	}

	replacer := strings.NewReplacer("[置顶]", "", "[原]", "", "[转]", "")
	title = strings.TrimSpace(replacer.Replace(title))

	contentSelection := doc.Find(rule.Content)
	content, err := contentSelection.Html()
	if err != nil {
		logger.Errorln("goquery parse content error:", err)
		return nil, err
	}
	content = strings.TrimSpace(content)
	txt := strings.TrimSpace(contentSelection.Text())
	txt = articleRe.ReplaceAllLiteralString(txt, " ")
	txt = articleSpaceRe.ReplaceAllLiteralString(txt, " ")

	// 自动抓取，内容长度不能少于 300 字
	if auto && len(txt) < 300 {
		logger.Infoln(articleUrl, "content is short")
		return nil, errors.New("content is short")
	}

	pubDate := util.TimeNow()
	if rule.PubDate != "" {
		pubDate = strings.TrimSpace(doc.Find(rule.PubDate).First().Text())

		// oschina patch
		re := regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}")
		submatches := re.FindStringSubmatch(pubDate)
		if len(submatches) > 0 {
			pubDate = submatches[0]
		}
	}

	if pubDate == "" {
		pubDate = util.TimeNow()
	} else {
		// YYYYY-MM-dd HH:mm
		if len(pubDate) == 16 && auto {
			// 三个月之前不入库
			pubTime := time.ParseInLocation("2006-01-02 15:04", pubDate, time.Local)
			if pubTime.Add(3 * 30 * 86400 * time.Second).Before(time.Now()) {
				return nil, errors.New("article is old!")
			}
		}
	}

	article := model.NewArticle()
	article.Domain = domain
	article.Name = rule.Name
	article.Author = author
	article.AuthorTxt = authorTxt
	article.Title = title
	article.Content = content
	article.Txt = txt
	article.PubDate = pubDate
	article.Url = articleUrl
	article.Lang = rule.Lang
	article.Ctime = util.TimeNow()

	_, err = article.Insert()
	if err != nil {
		logger.Errorln("insert article error:", err)
		return nil, err
	}

	return article, nil
}

// 获取抓取的文章列表（分页）
func FindArticleByPage(conds map[string]string, curPage, limit int) ([]*model.Article, int) {
	conditions := make([]string, 0, len(conds))
	for k, v := range conds {
		conditions = append(conditions, k+"="+v)
	}

	article := model.NewArticle()

	limitStr := strconv.Itoa((curPage-1)*limit) + "," + strconv.Itoa(limit)
	articleList, err := article.Where(strings.Join(conditions, " AND ")).Order("id DESC").Limit(limitStr).
		FindAll()
	if err != nil {
		logger.Errorln("article service FindArticleByPage Error:", err)
		return nil, 0
	}

	total, err := article.Count()
	if err != nil {
		logger.Errorln("article service FindArticleByPage COUNT Error:", err)
		return nil, 0
	}

	return articleList, total
}

// 获取抓取的文章列表（分页）
func FindArticles(lastId, limit string) []*model.Article {
	article := model.NewArticle()

	cond := "status IN(0,1)"
	if lastId != "0" {
		cond += " AND id<" + lastId
	}

	articleList, err := article.Where(cond).Order("id DESC").Limit(limit).
		FindAll()
	if err != nil {
		logger.Errorln("article service FindArticles Error:", err)
		return nil
	}

	topArticles, err := article.Where("top=?", 1).Order("id DESC").FindAll()
	if err != nil {
		logger.Errorln("article service Find Top Articles Error:", err)
		return nil
	}
	if len(topArticles) > 0 {
		articleList = append(topArticles, articleList...)
	}

	return articleList
}

// 获取单条博文
func FindArticleById(id string) (*model.Article, error) {
	article := model.NewArticle()
	err := article.Where("id=" + id).Find()
	if err != nil {
		logger.Errorln("article service FindArticleById Error:", err)
	}

	return article, err
}

// 获取当前(id)博文以及前后博文
func FindArticlesById(idstr string) (curArticle *model.Article, prevNext []*model.Article, err error) {

	id := util.MustInt(idstr)
	cond := "id BETWEEN ? AND ? AND status!=2"

	articles, err := model.NewArticle().Where(cond, id-5, id+5).FindAll()
	if err != nil {
		logger.Errorln("article service FindArticlesById Error:", err)
		return
	}

	if len(articles) == 0 {
		return
	}

	prevNext = make([]*model.Article, 2)
	prevId, nextId := articles[0].Id, articles[len(articles)-1].Id
	for _, article := range articles {
		if article.Id < id && article.Id > prevId {
			prevId = article.Id
			prevNext[0] = article
		} else if article.Id > id && article.Id < nextId {
			nextId = article.Id
			prevNext[1] = article
		} else if article.Id == id {
			curArticle = article
		}
	}

	if prevId == id {
		prevNext[0] = nil
	}

	if nextId == id {
		prevNext[1] = nil
	}

	return
}

// 获取多个文章详细信息
func FindArticlesByIds(ids []int) []*model.Article {
	if len(ids) == 0 {
		return nil
	}
	inIds := util.Join(ids, ",")
	articles, err := model.NewArticle().Where("id in(" + inIds + ")").FindAll()
	if err != nil {
		logger.Errorln("article service FindArticlesByIds error:", err)
		return nil
	}
	return articles
}

// 修改文章信息
func ModifyArticle(user map[string]interface{}, form url.Values) (errMsg string, err error) {

	username := user["username"].(string)
	form.Set("op_user", username)

	fields := []string{
		"title", "url", "cover", "author", "author_txt",
		"lang", "pub_date", "content",
		"tags", "status", "op_user",
	}
	query, args := updateSetClause(form, fields)

	id := form.Get("id")

	err = model.NewArticle().Set(query, args...).Where("id=" + id).Update()
	if err != nil {
		logger.Errorf("更新文章 【%s】 信息失败：%s\n", id, err)
		errMsg = "对不起，服务器内部错误，请稍后再试！"
		return
	}

	return
}

func DelArticle(id string) error {
	return model.NewArticle().Where("id=" + id).Delete()
}

// 博文总数
func ArticlesTotal() (total int) {
	total, err := model.NewArticle().Count()
	if err != nil {
		logger.Errorln("article service ArticlesTotal error:", err)
	}
	return
}

// 获取抓取规则列表（分页）
func FindRuleByPage(conds map[string]string, curPage, limit int) ([]*model.CrawlRule, int) {
	conditions := make([]string, 0, len(conds))
	for k, v := range conds {
		conditions = append(conditions, k+"="+v)
	}

	rule := model.NewCrawlRule()

	limitStr := strconv.Itoa((curPage-1)*limit) + "," + strconv.Itoa(limit)
	ruleList, err := rule.Where(strings.Join(conditions, " AND ")).Order("id DESC").Limit(limitStr).
		FindAll()
	if err != nil {
		logger.Errorln("rule service FindArticleByPage Error:", err)
		return nil, 0
	}

	total, err := rule.Count()
	if err != nil {
		logger.Errorln("rule service FindArticleByPage COUNT Error:", err)
		return nil, 0
	}

	return ruleList, total
}

func SaveRule(form url.Values, opUser string) (errMsg string, err error) {
	rule := model.NewCrawlRule()
	err = util.ConvertAssign(rule, form)
	if err != nil {
		logger.Errorln("rule ConvertAssign error", err)
		errMsg = err.Error()
		return
	}

	rule.OpUser = opUser

	if rule.Id != 0 {
		err = rule.Persist(rule)
	} else {
		_, err = rule.Insert()
	}

	if err != nil {
		errMsg = "内部服务器错误"
		logger.Errorln("rule save:", errMsg, ":", err)
		return
	}

	return
}

// 提供给其他service调用（包内）
func getArticles(ids map[int]int) map[int]*model.Article {
	articles := FindArticlesByIds(util.MapIntKeys(ids))
	articleMap := make(map[int]*model.Article, len(articles))
	for _, article := range articles {
		articleMap[article.Id] = article
	}
	return articleMap
}

// 博文评论
type ArticleComment struct{}

// 更新该文章的评论信息
// cid：评论id；objid：被评论对象id；uid：评论者；cmttime：评论时间
func (self ArticleComment) UpdateComment(cid, objid, uid int, cmttime string) {
	id := strconv.Itoa(objid)

	// 更新评论数（TODO：暂时每次都更新表）
	err := model.NewArticle().Where("id="+id).Increment("cmtnum", 1)
	if err != nil {
		logger.Errorln("更新文章评论数失败：", err)
	}
}

func (self ArticleComment) String() string {
	return "article"
}

// 实现 CommentObjecter 接口
func (self ArticleComment) SetObjinfo(ids []int, commentMap map[int][]*model.Comment) {
	articles := FindArticlesByIds(ids)
	if len(articles) == 0 {
		return
	}

	for _, article := range articles {
		objinfo := make(map[string]interface{})
		objinfo["title"] = article.Title
		objinfo["uri"] = model.PathUrlMap[model.TYPE_ARTICLE]
		objinfo["type_name"] = model.TypeNameMap[model.TYPE_ARTICLE]

		for _, comment := range commentMap[article.Id] {
			comment.Objinfo = objinfo
		}
	}
}

// 博文喜欢
type ArticleLike struct{}

// 更新该文章的喜欢数
// objid：被喜欢对象id；num: 喜欢数(负数表示取消喜欢)
func (self ArticleLike) UpdateLike(objid, num int) {
	// 更新喜欢数（TODO：暂时每次都更新表）
	err := model.NewArticle().Where("id=?", objid).Increment("likenum", num)
	if err != nil {
		logger.Errorln("更新文章喜欢数失败：", err)
	}
}

func (self ArticleLike) String() string {
	return "article"
}
<commit_msg>bugfix<commit_after>// Copyright 2014 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author：polaris	studygolang@gmail.com

package service

import (
	"errors"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"logger"
	"model"
	"util"

	"github.com/PuerkitoBio/goquery"
)

var domainPatch = map[string]string{
	"iteye.com":      "iteye.com",
	"blog.51cto.com": "blog.51cto.com",
}

var articleRe = regexp.MustCompile("[\r　\n  \t\v]+")
var articleSpaceRe = regexp.MustCompile("[ ]+")

// 获取url对应的文章并根据规则进行解析
func ParseArticle(articleUrl string, auto bool) (*model.Article, error) {
	articleUrl = strings.TrimSpace(articleUrl)
	if !strings.HasPrefix(articleUrl, "http") {
		articleUrl = "http://" + articleUrl
	}

	tmpArticle := model.NewArticle()
	err := tmpArticle.Where("url=" + articleUrl).Find("id")
	if err != nil || tmpArticle.Id != 0 {
		logger.Errorln(articleUrl, "has exists:", err)
		return nil, errors.New("has exists!")
	}

	urlPaths := strings.SplitN(articleUrl, "/", 5)
	domain := urlPaths[2]

	for k, v := range domainPatch {
		if strings.Contains(domain, k) && !strings.Contains(domain, "www."+k) {
			domain = v
			break
		}
	}

	rule := model.NewCrawlRule()
	err = rule.Where("domain=" + domain).Find()
	if err != nil {
		logger.Errorln("find rule by domain error:", err)
		return nil, err
	}

	if rule.Id == 0 {
		logger.Errorln("domain:", domain, "not exists!")
		return nil, errors.New("domain not exists")
	}

	var doc *goquery.Document
	if doc, err = goquery.NewDocument(articleUrl); err != nil {
		logger.Errorln("goquery newdocument error:", err)
		return nil, err
	}

	author, authorTxt := "", ""
	if rule.InUrl {
		index, err := strconv.Atoi(rule.Author)
		if err != nil {
			logger.Errorln("author rule is illegal:", rule.Author, "error:", err)
			return nil, err
		}
		author = urlPaths[index]
		authorTxt = author
	} else {
		if strings.HasPrefix(rule.Author, ".") || strings.HasPrefix(rule.Author, "#") {
			authorSelection := doc.Find(rule.Author)
			author, err = authorSelection.Html()
			if err != nil {
				logger.Errorln("goquery parse author error:", err)
				return nil, err
			}

			author = strings.TrimSpace(author)
			authorTxt = strings.TrimSpace(authorSelection.Text())
		} else {
			// 某些个人博客，页面中没有作者的信息，因此，规则中 author 即为 作者
			author = rule.Author
			authorTxt = rule.Author
		}
	}

	title := ""
	doc.Find(rule.Title).Each(func(i int, selection *goquery.Selection) {
		if title != "" {
			return
		}

		tmpTitle := strings.TrimSpace(strings.TrimPrefix(selection.Text(), "原"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "荐"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "转"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "顶"))
		if tmpTitle != "" {
			title = tmpTitle
		}
	})

	if title == "" {
		logger.Errorln("url:", articleUrl, "parse title error:", err)
		return nil, err
	}

	replacer := strings.NewReplacer("[置顶]", "", "[原]", "", "[转]", "")
	title = strings.TrimSpace(replacer.Replace(title))

	contentSelection := doc.Find(rule.Content)
	content, err := contentSelection.Html()
	if err != nil {
		logger.Errorln("goquery parse content error:", err)
		return nil, err
	}
	content = strings.TrimSpace(content)
	txt := strings.TrimSpace(contentSelection.Text())
	txt = articleRe.ReplaceAllLiteralString(txt, " ")
	txt = articleSpaceRe.ReplaceAllLiteralString(txt, " ")

	// 自动抓取，内容长度不能少于 300 字
	if auto && len(txt) < 300 {
		logger.Infoln(articleUrl, "content is short")
		return nil, errors.New("content is short")
	}

	pubDate := util.TimeNow()
	if rule.PubDate != "" {
		pubDate = strings.TrimSpace(doc.Find(rule.PubDate).First().Text())

		// oschina patch
		re := regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}")
		submatches := re.FindStringSubmatch(pubDate)
		if len(submatches) > 0 {
			pubDate = submatches[0]
		}
	}

	if pubDate == "" {
		pubDate = util.TimeNow()
	} else {
		// YYYYY-MM-dd HH:mm
		if len(pubDate) == 16 && auto {
			// 三个月之前不入库
			pubTime, err := time.ParseInLocation("2006-01-02 15:04", pubDate, time.Local)
			if err == nil {
				if pubTime.Add(3 * 30 * 86400 * time.Second).Before(time.Now()) {
					return nil, errors.New("article is old!")
				}
			}
		}
	}

	article := model.NewArticle()
	article.Domain = domain
	article.Name = rule.Name
	article.Author = author
	article.AuthorTxt = authorTxt
	article.Title = title
	article.Content = content
	article.Txt = txt
	article.PubDate = pubDate
	article.Url = articleUrl
	article.Lang = rule.Lang
	article.Ctime = util.TimeNow()

	_, err = article.Insert()
	if err != nil {
		logger.Errorln("insert article error:", err)
		return nil, err
	}

	return article, nil
}

// 获取抓取的文章列表（分页）
func FindArticleByPage(conds map[string]string, curPage, limit int) ([]*model.Article, int) {
	conditions := make([]string, 0, len(conds))
	for k, v := range conds {
		conditions = append(conditions, k+"="+v)
	}

	article := model.NewArticle()

	limitStr := strconv.Itoa((curPage-1)*limit) + "," + strconv.Itoa(limit)
	articleList, err := article.Where(strings.Join(conditions, " AND ")).Order("id DESC").Limit(limitStr).
		FindAll()
	if err != nil {
		logger.Errorln("article service FindArticleByPage Error:", err)
		return nil, 0
	}

	total, err := article.Count()
	if err != nil {
		logger.Errorln("article service FindArticleByPage COUNT Error:", err)
		return nil, 0
	}

	return articleList, total
}

// 获取抓取的文章列表（分页）
func FindArticles(lastId, limit string) []*model.Article {
	article := model.NewArticle()

	cond := "status IN(0,1)"
	if lastId != "0" {
		cond += " AND id<" + lastId
	}

	articleList, err := article.Where(cond).Order("id DESC").Limit(limit).
		FindAll()
	if err != nil {
		logger.Errorln("article service FindArticles Error:", err)
		return nil
	}

	topArticles, err := article.Where("top=?", 1).Order("id DESC").FindAll()
	if err != nil {
		logger.Errorln("article service Find Top Articles Error:", err)
		return nil
	}
	if len(topArticles) > 0 {
		articleList = append(topArticles, articleList...)
	}

	return articleList
}

// 获取单条博文
func FindArticleById(id string) (*model.Article, error) {
	article := model.NewArticle()
	err := article.Where("id=" + id).Find()
	if err != nil {
		logger.Errorln("article service FindArticleById Error:", err)
	}

	return article, err
}

// 获取当前(id)博文以及前后博文
func FindArticlesById(idstr string) (curArticle *model.Article, prevNext []*model.Article, err error) {

	id := util.MustInt(idstr)
	cond := "id BETWEEN ? AND ? AND status!=2"

	articles, err := model.NewArticle().Where(cond, id-5, id+5).FindAll()
	if err != nil {
		logger.Errorln("article service FindArticlesById Error:", err)
		return
	}

	if len(articles) == 0 {
		return
	}

	prevNext = make([]*model.Article, 2)
	prevId, nextId := articles[0].Id, articles[len(articles)-1].Id
	for _, article := range articles {
		if article.Id < id && article.Id > prevId {
			prevId = article.Id
			prevNext[0] = article
		} else if article.Id > id && article.Id < nextId {
			nextId = article.Id
			prevNext[1] = article
		} else if article.Id == id {
			curArticle = article
		}
	}

	if prevId == id {
		prevNext[0] = nil
	}

	if nextId == id {
		prevNext[1] = nil
	}

	return
}

// 获取多个文章详细信息
func FindArticlesByIds(ids []int) []*model.Article {
	if len(ids) == 0 {
		return nil
	}
	inIds := util.Join(ids, ",")
	articles, err := model.NewArticle().Where("id in(" + inIds + ")").FindAll()
	if err != nil {
		logger.Errorln("article service FindArticlesByIds error:", err)
		return nil
	}
	return articles
}

// 修改文章信息
func ModifyArticle(user map[string]interface{}, form url.Values) (errMsg string, err error) {

	username := user["username"].(string)
	form.Set("op_user", username)

	fields := []string{
		"title", "url", "cover", "author", "author_txt",
		"lang", "pub_date", "content",
		"tags", "status", "op_user",
	}
	query, args := updateSetClause(form, fields)

	id := form.Get("id")

	err = model.NewArticle().Set(query, args...).Where("id=" + id).Update()
	if err != nil {
		logger.Errorf("更新文章 【%s】 信息失败：%s\n", id, err)
		errMsg = "对不起，服务器内部错误，请稍后再试！"
		return
	}

	return
}

func DelArticle(id string) error {
	return model.NewArticle().Where("id=" + id).Delete()
}

// 博文总数
func ArticlesTotal() (total int) {
	total, err := model.NewArticle().Count()
	if err != nil {
		logger.Errorln("article service ArticlesTotal error:", err)
	}
	return
}

// 获取抓取规则列表（分页）
func FindRuleByPage(conds map[string]string, curPage, limit int) ([]*model.CrawlRule, int) {
	conditions := make([]string, 0, len(conds))
	for k, v := range conds {
		conditions = append(conditions, k+"="+v)
	}

	rule := model.NewCrawlRule()

	limitStr := strconv.Itoa((curPage-1)*limit) + "," + strconv.Itoa(limit)
	ruleList, err := rule.Where(strings.Join(conditions, " AND ")).Order("id DESC").Limit(limitStr).
		FindAll()
	if err != nil {
		logger.Errorln("rule service FindArticleByPage Error:", err)
		return nil, 0
	}

	total, err := rule.Count()
	if err != nil {
		logger.Errorln("rule service FindArticleByPage COUNT Error:", err)
		return nil, 0
	}

	return ruleList, total
}

func SaveRule(form url.Values, opUser string) (errMsg string, err error) {
	rule := model.NewCrawlRule()
	err = util.ConvertAssign(rule, form)
	if err != nil {
		logger.Errorln("rule ConvertAssign error", err)
		errMsg = err.Error()
		return
	}

	rule.OpUser = opUser

	if rule.Id != 0 {
		err = rule.Persist(rule)
	} else {
		_, err = rule.Insert()
	}

	if err != nil {
		errMsg = "内部服务器错误"
		logger.Errorln("rule save:", errMsg, ":", err)
		return
	}

	return
}

// 提供给其他service调用（包内）
func getArticles(ids map[int]int) map[int]*model.Article {
	articles := FindArticlesByIds(util.MapIntKeys(ids))
	articleMap := make(map[int]*model.Article, len(articles))
	for _, article := range articles {
		articleMap[article.Id] = article
	}
	return articleMap
}

// 博文评论
type ArticleComment struct{}

// 更新该文章的评论信息
// cid：评论id；objid：被评论对象id；uid：评论者；cmttime：评论时间
func (self ArticleComment) UpdateComment(cid, objid, uid int, cmttime string) {
	id := strconv.Itoa(objid)

	// 更新评论数（TODO：暂时每次都更新表）
	err := model.NewArticle().Where("id="+id).Increment("cmtnum", 1)
	if err != nil {
		logger.Errorln("更新文章评论数失败：", err)
	}
}

func (self ArticleComment) String() string {
	return "article"
}

// 实现 CommentObjecter 接口
func (self ArticleComment) SetObjinfo(ids []int, commentMap map[int][]*model.Comment) {
	articles := FindArticlesByIds(ids)
	if len(articles) == 0 {
		return
	}

	for _, article := range articles {
		objinfo := make(map[string]interface{})
		objinfo["title"] = article.Title
		objinfo["uri"] = model.PathUrlMap[model.TYPE_ARTICLE]
		objinfo["type_name"] = model.TypeNameMap[model.TYPE_ARTICLE]

		for _, comment := range commentMap[article.Id] {
			comment.Objinfo = objinfo
		}
	}
}

// 博文喜欢
type ArticleLike struct{}

// 更新该文章的喜欢数
// objid：被喜欢对象id；num: 喜欢数(负数表示取消喜欢)
func (self ArticleLike) UpdateLike(objid, num int) {
	// 更新喜欢数（TODO：暂时每次都更新表）
	err := model.NewArticle().Where("id=?", objid).Increment("likenum", num)
	if err != nil {
		logger.Errorln("更新文章喜欢数失败：", err)
	}
}

func (self ArticleLike) String() string {
	return "article"
}
<|endoftext|>


<commit_before>package libgobuster

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"unicode/utf8"

	uuid "github.com/satori/go.uuid"
)

type RedirectHandler struct {
	Transport http.RoundTripper
	State     *State
}

type RedirectError struct {
	StatusCode int
}

func (e *RedirectError) Error() string {
	return fmt.Sprintf("Redirect code: %d", e.StatusCode)
}

func (rh *RedirectHandler) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	if rh.State.FollowRedirect {
		return rh.Transport.RoundTrip(req)
	}

	resp, err = rh.Transport.RoundTrip(req)
	if err != nil {
		return resp, err
	}

	switch resp.StatusCode {
	case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
		http.StatusNotModified, http.StatusUseProxy, http.StatusTemporaryRedirect:
		return nil, &RedirectError{StatusCode: resp.StatusCode}
	}

	return resp, err
}

// Make a request to the given URL.
func MakeRequest(s *State, fullUrl, cookie string) (*int, *int64) {
	req, err := http.NewRequest("GET", fullUrl, nil)

	if err != nil {
		return nil, nil
	}

	if cookie != "" {
		req.Header.Set("Cookie", cookie)
	}

	if s.UserAgent != "" {
		req.Header.Set("User-Agent", s.UserAgent)
	}

	if s.Username != "" {
		req.SetBasicAuth(s.Username, s.Password)
	}

	resp, err := s.Client.Do(req)

	if err != nil {
		if ue, ok := err.(*url.Error); ok {

			if strings.HasPrefix(ue.Err.Error(), "x509") {
				fmt.Println("[-] Invalid certificate")
			}

			if re, ok := ue.Err.(*RedirectError); ok {
				return &re.StatusCode, nil
			}
		}
		return nil, nil
	}

	defer resp.Body.Close()

	var length *int64 = nil

	if s.IncludeLength {
		length = new(int64)
		if resp.ContentLength <= 0 {
			body, err := ioutil.ReadAll(resp.Body)
			if err == nil {
				*length = int64(utf8.RuneCountInString(string(body)))
			}
		} else {
			*length = resp.ContentLength
		}
	}

	return &resp.StatusCode, length
}

// Small helper to combine URL with URI then make a
// request to the generated location.
func GoGet(s *State, url, uri, cookie string) (*int, *int64) {
	return MakeRequest(s, url+uri, cookie)
}

func SetupDir(s *State) bool {
	guid := uuid.Must(uuid.NewV4())
	wildcardResp, _ := GoGet(s, s.Url, fmt.Sprintf("%s", guid), s.Cookies)

	if s.StatusCodes.Contains(*wildcardResp) {
		s.IsWildcard = true
		fmt.Println("[-] Wildcard response found:", fmt.Sprintf("%s%s", s.Url, guid), "=>", *wildcardResp)
		if !s.WildcardForced {
			fmt.Println("[-] To force processing of Wildcard responses, specify the '-fw' switch.")
		}
		return s.WildcardForced
	}

	return true
}

func ProcessDirEntry(s *State, word string, resultChan chan<- Result) {
	suffix := ""
	if s.UseSlash {
		suffix = "/"
	}

	// Try the DIR first
	dirResp, dirSize := GoGet(s, s.Url, word+suffix, s.Cookies)
	if dirResp != nil {
		resultChan <- Result{
			Entity: word + suffix,
			Status: *dirResp,
			Size:   dirSize,
		}
	}

	// Follow up with files using each ext.
	for ext := range s.Extensions {
		file := word + s.Extensions[ext]
		fileResp, fileSize := GoGet(s, s.Url, file, s.Cookies)

		if fileResp != nil {
			resultChan <- Result{
				Entity: file,
				Status: *fileResp,
				Size:   fileSize,
			}
		}
	}
}

func PrintDirResult(s *State, r *Result) {
	output := ""

	// Prefix if we're in verbose mode
	if s.Verbose {
		if s.StatusCodes.Contains(r.Status) {
			output = "Found : "
		} else {
			output = "Missed: "
		}
	}

	if s.StatusCodes.Contains(r.Status) || s.Verbose {
		if s.Expanded {
			output += s.Url
		} else {
			output += "/"
		}
		output += r.Entity

		if !s.NoStatus {
			output += fmt.Sprintf(" (Status: %d)", r.Status)
		}

		if r.Size != nil {
			output += fmt.Sprintf(" [Size: %d]", *r.Size)
		}
		output += "\n"

		fmt.Printf(output)

		if s.OutputFile != nil {
			WriteToFile(output, s)
		}
	}
}
<commit_msg>fix connection reuse<commit_after>package libgobuster

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"unicode/utf8"

	uuid "github.com/satori/go.uuid"
)

type RedirectHandler struct {
	Transport http.RoundTripper
	State     *State
}

type RedirectError struct {
	StatusCode int
}

func (e *RedirectError) Error() string {
	return fmt.Sprintf("Redirect code: %d", e.StatusCode)
}

func (rh *RedirectHandler) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	if rh.State.FollowRedirect {
		return rh.Transport.RoundTrip(req)
	}

	resp, err = rh.Transport.RoundTrip(req)
	if err != nil {
		return resp, err
	}

	switch resp.StatusCode {
	case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
		http.StatusNotModified, http.StatusUseProxy, http.StatusTemporaryRedirect:
		return nil, &RedirectError{StatusCode: resp.StatusCode}
	}

	return resp, err
}

// Make a request to the given URL.
func MakeRequest(s *State, fullUrl, cookie string) (*int, *int64) {
	req, err := http.NewRequest("GET", fullUrl, nil)

	if err != nil {
		return nil, nil
	}

	if cookie != "" {
		req.Header.Set("Cookie", cookie)
	}

	if s.UserAgent != "" {
		req.Header.Set("User-Agent", s.UserAgent)
	}

	if s.Username != "" {
		req.SetBasicAuth(s.Username, s.Password)
	}

	resp, err := s.Client.Do(req)

	if err != nil {
		if ue, ok := err.(*url.Error); ok {

			if strings.HasPrefix(ue.Err.Error(), "x509") {
				fmt.Println("[-] Invalid certificate")
			}

			if re, ok := ue.Err.(*RedirectError); ok {
				return &re.StatusCode, nil
			}
		}
		return nil, nil
	}

	defer resp.Body.Close()

	var length *int64 = nil

	if s.IncludeLength {
		length = new(int64)
		if resp.ContentLength <= 0 {
			body, err := ioutil.ReadAll(resp.Body)
			if err == nil {
				*length = int64(utf8.RuneCountInString(string(body)))
			}
		} else {
			*length = resp.ContentLength
		}
	} else {
		// DO NOT REMOVE!
		// absolutely needed so golang will reuse connections!
		_, err = io.Copy(ioutil.Discard, resp.Body)
		if err != nil {
			return nil, nil
		}
	}

	return &resp.StatusCode, length
}

// Small helper to combine URL with URI then make a
// request to the generated location.
func GoGet(s *State, url, uri, cookie string) (*int, *int64) {
	return MakeRequest(s, url+uri, cookie)
}

func SetupDir(s *State) bool {
	guid := uuid.Must(uuid.NewV4())
	wildcardResp, _ := GoGet(s, s.Url, fmt.Sprintf("%s", guid), s.Cookies)

	if s.StatusCodes.Contains(*wildcardResp) {
		s.IsWildcard = true
		fmt.Println("[-] Wildcard response found:", fmt.Sprintf("%s%s", s.Url, guid), "=>", *wildcardResp)
		if !s.WildcardForced {
			fmt.Println("[-] To force processing of Wildcard responses, specify the '-fw' switch.")
		}
		return s.WildcardForced
	}

	return true
}

func ProcessDirEntry(s *State, word string, resultChan chan<- Result) {
	suffix := ""
	if s.UseSlash {
		suffix = "/"
	}

	// Try the DIR first
	dirResp, dirSize := GoGet(s, s.Url, word+suffix, s.Cookies)
	if dirResp != nil {
		resultChan <- Result{
			Entity: word + suffix,
			Status: *dirResp,
			Size:   dirSize,
		}
	}

	// Follow up with files using each ext.
	for ext := range s.Extensions {
		file := word + s.Extensions[ext]
		fileResp, fileSize := GoGet(s, s.Url, file, s.Cookies)

		if fileResp != nil {
			resultChan <- Result{
				Entity: file,
				Status: *fileResp,
				Size:   fileSize,
			}
		}
	}
}

func PrintDirResult(s *State, r *Result) {
	output := ""

	// Prefix if we're in verbose mode
	if s.Verbose {
		if s.StatusCodes.Contains(r.Status) {
			output = "Found : "
		} else {
			output = "Missed: "
		}
	}

	if s.StatusCodes.Contains(r.Status) || s.Verbose {
		if s.Expanded {
			output += s.Url
		} else {
			output += "/"
		}
		output += r.Entity

		if !s.NoStatus {
			output += fmt.Sprintf(" (Status: %d)", r.Status)
		}

		if r.Size != nil {
			output += fmt.Sprintf(" [Size: %d]", *r.Size)
		}
		output += "\n"

		fmt.Printf(output)

		if s.OutputFile != nil {
			WriteToFile(output, s)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package bloom_test

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/conformal/btcutil/bloom"
	"github.com/conformal/btcwire"
)

// This example creates a new bloom filter, adds a transaction hash to it, and
// shows how to check if the filter matches the transaction.
func ExampleNewFilter() {
	rand.Seed(time.Now().UnixNano())
	tweak := rand.Uint32()

	// Create a new bloom filter intended to hold 10 elements with a 0.01%
	// false positive rate and does not include any automatic update
	// functionality when transactions are matched.
	filter := bloom.NewFilter(10, tweak, 0.0001, btcwire.BloomUpdateNone)

	// Create a transaction hash and add it to the filter.  This particular
	// trasaction is the first transaction in block 310,000 of the main
	// bitcoin block chain.
	txHashStr := "fd611c56ca0d378cdcd16244b45c2ba9588da3adac367c4ef43e808b280b8a45"
	txHash, err := btcwire.NewShaHashFromStr(txHashStr)
	if err != nil {
		fmt.Println(err)
		return
	}
	filter.AddShaHash(txHash)

	// Show that the filter matches.
	matches := filter.Matches(txHash.Bytes())
	fmt.Println("Filter Matches?:", matches)

	// Output:
	// Filter Matches?: true
}
<commit_msg>Add READEME.md for bloom package.<commit_after>// Copyright (c) 2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package bloom_test

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/conformal/btcutil/bloom"
	"github.com/conformal/btcwire"
)

// This example demonstrates how to create a new bloom filter, add a transaction
// hash to it, and check if the filter matches the transaction.
func ExampleNewFilter() {
	rand.Seed(time.Now().UnixNano())
	tweak := rand.Uint32()

	// Create a new bloom filter intended to hold 10 elements with a 0.01%
	// false positive rate and does not include any automatic update
	// functionality when transactions are matched.
	filter := bloom.NewFilter(10, tweak, 0.0001, btcwire.BloomUpdateNone)

	// Create a transaction hash and add it to the filter.  This particular
	// trasaction is the first transaction in block 310,000 of the main
	// bitcoin block chain.
	txHashStr := "fd611c56ca0d378cdcd16244b45c2ba9588da3adac367c4ef43e808b280b8a45"
	txHash, err := btcwire.NewShaHashFromStr(txHashStr)
	if err != nil {
		fmt.Println(err)
		return
	}
	filter.AddShaHash(txHash)

	// Show that the filter matches.
	matches := filter.Matches(txHash.Bytes())
	fmt.Println("Filter Matches?:", matches)

	// Output:
	// Filter Matches?: true
}
<|endoftext|>