<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	goopt "github.com/droundy/goopt"
	colors "github.com/wsxiaoys/colors"
	"./ignore"
	"os"
	"path/filepath"
	"regexp"
)

var Author = "Alexander Solovyov"
var Version = "0.3.3"
var Summary = "gr [OPTS] string-to-search\n"

var byteNewLine []byte = []byte("\n")

var onlyName = goopt.Flag([]string{"-n", "--filename"}, []string{},
	"print only filenames", "")
var ignoreFiles = goopt.Strings([]string{"-x", "--exclude"}, "RE",
	"exclude files that match the regexp from search")
var singleline = goopt.Flag([]string{"-s", "--singleline"}, []string{},
	"match on a single line (^/$ will be beginning/end of line)", "")
var replace = goopt.String([]string{"-r", "--replace"}, "",
	"replace found substrings with this string")
var force = goopt.Flag([]string{"--force"}, []string{},
	"force replacement in binary files", "")
var showVersion = goopt.Flag([]string{"-v", "--version"}, []string{},
	"show version and exit", "")

func main() {
	goopt.Author = Author
	goopt.Version = Version
	goopt.Summary = Summary
	goopt.Usage = func() string {
		return fmt.Sprintf("Usage of goreplace %s:\n\t", Version) +
			goopt.Summary + "\n" + goopt.Help()
	}

	cwd, _ := os.Getwd()
	ignorer := ignore.New(cwd)
	goopt.Summary += fmt.Sprintf("\n%s", ignorer)

	goopt.Parse(nil)

	if *showVersion {
		println("goreplace " + goopt.Version)
		return
	}

	ignorer.Append(*ignoreFiles)

	if len(goopt.Args) == 0 {
		println(goopt.Usage())
		return
	}

	pattern, err := regexp.Compile(goopt.Args[0])
	errhandle(err, true, "can't compile regexp %s", goopt.Args[0])

	searchFiles(pattern, ignorer)
}

func errhandle(err error, exit bool, moreinfo string, a ...interface{}) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "ERR %s\n%s\n", err, fmt.Sprintf(moreinfo, a...))
	if exit {
		os.Exit(1)
	}
}

func searchFiles(pattern *regexp.Regexp, ignorer ignore.Ignorer) {
	v := &GRVisitor{pattern, ignorer, false}

	errors := make(chan error, 64)

	filepath.Walk(".", walkFunc(v, errors))

	select {
	case err := <-errors:
		errhandle(err, true, "some error")
	default:
	}
}

func walkFunc(v *GRVisitor, errors chan<- error) filepath.WalkFunc {
	return func(fn string, fi os.FileInfo, err error) error {
		if err != nil {
			errors <- err
			return nil
		}

		// NOTE: if a directory is a symlink, filepath.Walk won't recurse inside
		if fi.Mode() & os.ModeSymlink != 0 {
			fi, err = os.Stat(fn)
			if err != nil {
				errors <- err
				return nil
			}
		}

		if fi.IsDir() {
			if !v.VisitDir(fn, fi) {
				return filepath.SkipDir
			}
			return nil
		}

		v.VisitFile(fn, fi)
		return nil
	}
}

type GRVisitor struct {
	pattern *regexp.Regexp
	ignorer ignore.Ignorer
	// Used to prevent sparse newline at the end of output
	prependNewLine bool
}

func (v *GRVisitor) VisitDir(fn string, fi os.FileInfo) bool {
	return !v.ignorer.Ignore(fi.Name(), true)
}

func (v *GRVisitor) VisitFile(fn string, fi os.FileInfo) {
	if fi.IsDir() {
		return
	}

	if fi.Size() >= 1024*1024*10 {
		fmt.Fprintf(os.Stderr, "Skipping %s, too big: %d\n", fn, fi.Size())
		return
	}

	if fi.Size() == 0 {
		return
	}

	if v.ignorer.Ignore(fn, false) {
		return
	}

	f, content := v.GetFileAndContent(fn, fi)
	defer f.Close()

	if len(*replace) == 0 {
		v.SearchFile(fn, content)
		return
	}

	changed, result := v.ReplaceInFile(fn, content)
	if changed {
		f.Seek(0, 0)
		n, err := f.Write(result)
		errhandle(err, true, "Error writing replacement in file %s", fn)
		if int64(n) < fi.Size() {
			err := f.Truncate(int64(n))
			errhandle(err, true, "Error truncating file to size %d", f)
		}
	}
}

func (v *GRVisitor) GetFileAndContent(fn string, fi os.FileInfo) (f *os.File, content []byte) {
	var err error
	var msg string

	if len(*replace) > 0 {
		f, err = os.OpenFile(fn, os.O_RDWR, 0666)
		msg = "can't open file %s for reading and writing"
	} else {
		f, err = os.Open(fn)
		msg = "can't open file %s for reading"
	}

	if err != nil {
		errhandle(err, false, msg, fn)
		return
	}

	content = make([]byte, fi.Size())
	n, err := f.Read(content)
	errhandle(err, true, "can't read file %s", fn)
	if int64(n) != fi.Size() {
		panic(fmt.Sprintf("Not whole file was read, only %d from %d",
			n, fi.Size()))
	}

	return
}

func (v *GRVisitor) SearchFile(fn string, content []byte) {
	lines := IntList([]int{})
	binary := false

	if bytes.IndexByte(content, 0) != -1 {
		binary = true
	}

	for _, info := range v.FindAllIndex(content) {
		if lines.Contains(info.num) {
			continue
		}

		if v.prependNewLine {
			fmt.Println("")
			v.prependNewLine = false
		}

		var first = len(lines) == 0
		lines = append(lines, info.num)

		if first {
			if binary && !*onlyName {
				fmt.Printf("Binary file %s matches\n", fn)
				break
			} else {
				colors.Printf("@g%s\n", fn)
			}
		}

		if *onlyName {
			return
		}

		colors.Printf("@!@y%d:", info.num)
		coloredLine := v.pattern.ReplaceAllStringFunc(string(info.line),
			func(wrap string) string {
			return colors.Sprintf("@Y%s", wrap)
		})
		fmt.Printf("%s\n", coloredLine)
	}

	if len(lines) > 0 {
		v.prependNewLine = true
	}
}

func getSuffix(num int) string {
	if num > 1 {
		return "s"
	}
	return ""
}

func (v *GRVisitor) ReplaceInFile(fn string, content []byte) (changed bool, result []byte) {
	changed = false
	binary := false
	changenum := 0

	if *singleline {
		panic("Can't handle singleline replacements yet")
	}

	if bytes.IndexByte(content, 0) != -1 {
		binary = true
	}

	result = v.pattern.ReplaceAllFunc(content, func(s []byte) []byte {
		if binary && !*force {
			errhandle(
				errors.New("supply --force to force change of binary file"),
				false, "")
		}
		if !changed {
			changed = true
			colors.Printf("@g%s", fn)
		}

		changenum += 1
		return []byte(*replace)
	})

	if changenum > 0 {
		colors.Printf("@!@y - %d change%s made\n",
			changenum, getSuffix(changenum))
	}

	return changed, result
}

type LineInfo struct {
	num  int
	line []byte
}

// will return slice of [linenum, line] slices
func (v *GRVisitor) FindAllIndex(content []byte) (res []*LineInfo) {
	linenum := 1

	if *singleline {
		begin, end := 0, 0
		for i := 0; i < len(content); i++ {
			if content[i] == '\n' {
				end = i
				line := content[begin:end]
				if v.pattern.Match(line) {
					res = append(res, &LineInfo{linenum, line})
				}
				linenum += 1
				begin = end + 1
			}
		}
		return res
	}

	last := 0
	for _, bounds := range v.pattern.FindAllIndex(content, -1) {
		linenum += bytes.Count(content[last:bounds[0]], byteNewLine)
		last = bounds[0]
		begin, end := beginend(content, bounds[0], bounds[1])
		res = append(res, &LineInfo{linenum, content[begin:end]})
	}
	return res
}

// Given a []byte, start and finish of some inner slice, will find nearest
// newlines on both ends of this slice
func beginend(s []byte, start int, finish int) (begin int, end int) {
	begin = 0
	end = len(s)

	for i := start; i >= 0; i-- {
		if s[i] == byteNewLine[0] {
			begin = i + 1
			break
		}
	}

	// -1 to check if current location is not end of string
	for i := finish - 1; i < len(s); i++ {
		if s[i] == byteNewLine[0] {
			end = i
			break
		}
	}

	return
}

type IntList []int

func (il IntList) Contains(i int) bool {
	for _, x := range il {
		if x == i {
			return true
		}
	}
	return false
}
<commit_msg>use if initialization when possible :)<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	goopt "github.com/droundy/goopt"
	colors "github.com/wsxiaoys/colors"
	"./ignore"
	"os"
	"path/filepath"
	"regexp"
)

var Author = "Alexander Solovyov"
var Version = "0.3.3"
var Summary = "gr [OPTS] string-to-search\n"

var byteNewLine []byte = []byte("\n")

var onlyName = goopt.Flag([]string{"-n", "--filename"}, []string{},
	"print only filenames", "")
var ignoreFiles = goopt.Strings([]string{"-x", "--exclude"}, "RE",
	"exclude files that match the regexp from search")
var singleline = goopt.Flag([]string{"-s", "--singleline"}, []string{},
	"match on a single line (^/$ will be beginning/end of line)", "")
var replace = goopt.String([]string{"-r", "--replace"}, "",
	"replace found substrings with this string")
var force = goopt.Flag([]string{"--force"}, []string{},
	"force replacement in binary files", "")
var showVersion = goopt.Flag([]string{"-v", "--version"}, []string{},
	"show version and exit", "")

func main() {
	goopt.Author = Author
	goopt.Version = Version
	goopt.Summary = Summary
	goopt.Usage = func() string {
		return fmt.Sprintf("Usage of goreplace %s:\n\t", Version) +
			goopt.Summary + "\n" + goopt.Help()
	}

	cwd, _ := os.Getwd()
	ignorer := ignore.New(cwd)
	goopt.Summary += fmt.Sprintf("\n%s", ignorer)

	goopt.Parse(nil)

	if *showVersion {
		println("goreplace " + goopt.Version)
		return
	}

	ignorer.Append(*ignoreFiles)

	if len(goopt.Args) == 0 {
		println(goopt.Usage())
		return
	}

	pattern, err := regexp.Compile(goopt.Args[0])
	errhandle(err, true, "can't compile regexp %s", goopt.Args[0])

	searchFiles(pattern, ignorer)
}

func errhandle(err error, exit bool, moreinfo string, a ...interface{}) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "ERR %s\n%s\n", err, fmt.Sprintf(moreinfo, a...))
	if exit {
		os.Exit(1)
	}
}

func searchFiles(pattern *regexp.Regexp, ignorer ignore.Ignorer) {
	v := &GRVisitor{pattern, ignorer, false}

	errors := make(chan error, 64)

	filepath.Walk(".", walkFunc(v, errors))

	select {
	case err := <-errors:
		errhandle(err, true, "some error")
	default:
	}
}

func walkFunc(v *GRVisitor, errors chan<- error) filepath.WalkFunc {
	return func(fn string, fi os.FileInfo, err error) error {
		if err != nil {
			errors <- err
			return nil
		}

		// NOTE: if a directory is a symlink, filepath.Walk won't recurse inside
		if fi.Mode() & os.ModeSymlink != 0 {
			if fi, err = os.Stat(fn); err != nil {
				errors <- err
				return nil
			}
		}

		if fi.IsDir() {
			if !v.VisitDir(fn, fi) {
				return filepath.SkipDir
			}
			return nil
		}

		v.VisitFile(fn, fi)
		return nil
	}
}

type GRVisitor struct {
	pattern *regexp.Regexp
	ignorer ignore.Ignorer
	// Used to prevent sparse newline at the end of output
	prependNewLine bool
}

func (v *GRVisitor) VisitDir(fn string, fi os.FileInfo) bool {
	return !v.ignorer.Ignore(fi.Name(), true)
}

func (v *GRVisitor) VisitFile(fn string, fi os.FileInfo) {
	if fi.IsDir() {
		return
	}

	if fi.Size() >= 1024*1024*10 {
		fmt.Fprintf(os.Stderr, "Skipping %s, too big: %d\n", fn, fi.Size())
		return
	}

	if fi.Size() == 0 {
		return
	}

	if v.ignorer.Ignore(fn, false) {
		return
	}

	f, content := v.GetFileAndContent(fn, fi)
	defer f.Close()

	if len(*replace) == 0 {
		v.SearchFile(fn, content)
		return
	}

	changed, result := v.ReplaceInFile(fn, content)
	if changed {
		f.Seek(0, 0)
		n, err := f.Write(result)
		errhandle(err, true, "Error writing replacement in file %s", fn)
		if int64(n) < fi.Size() {
			err := f.Truncate(int64(n))
			errhandle(err, true, "Error truncating file to size %d", f)
		}
	}
}

func (v *GRVisitor) GetFileAndContent(fn string, fi os.FileInfo) (f *os.File, content []byte) {
	var err error
	var msg string

	if len(*replace) > 0 {
		f, err = os.OpenFile(fn, os.O_RDWR, 0666)
		msg = "can't open file %s for reading and writing"
	} else {
		f, err = os.Open(fn)
		msg = "can't open file %s for reading"
	}

	if err != nil {
		errhandle(err, false, msg, fn)
		return
	}

	content = make([]byte, fi.Size())
	n, err := f.Read(content)
	errhandle(err, true, "can't read file %s", fn)
	if int64(n) != fi.Size() {
		panic(fmt.Sprintf("Not whole file was read, only %d from %d",
			n, fi.Size()))
	}

	return
}

func (v *GRVisitor) SearchFile(fn string, content []byte) {
	lines := IntList([]int{})
	binary := false

	if bytes.IndexByte(content, 0) != -1 {
		binary = true
	}

	for _, info := range v.FindAllIndex(content) {
		if lines.Contains(info.num) {
			continue
		}

		if v.prependNewLine {
			fmt.Println("")
			v.prependNewLine = false
		}

		var first = len(lines) == 0
		lines = append(lines, info.num)

		if first {
			if binary && !*onlyName {
				fmt.Printf("Binary file %s matches\n", fn)
				break
			} else {
				colors.Printf("@g%s\n", fn)
			}
		}

		if *onlyName {
			return
		}

		colors.Printf("@!@y%d:", info.num)
		coloredLine := v.pattern.ReplaceAllStringFunc(string(info.line),
			func(wrap string) string {
			return colors.Sprintf("@Y%s", wrap)
		})
		fmt.Printf("%s\n", coloredLine)
	}

	if len(lines) > 0 {
		v.prependNewLine = true
	}
}

func getSuffix(num int) string {
	if num > 1 {
		return "s"
	}
	return ""
}

func (v *GRVisitor) ReplaceInFile(fn string, content []byte) (changed bool, result []byte) {
	changed = false
	binary := false
	changenum := 0

	if *singleline {
		panic("Can't handle singleline replacements yet")
	}

	if bytes.IndexByte(content, 0) != -1 {
		binary = true
	}

	result = v.pattern.ReplaceAllFunc(content, func(s []byte) []byte {
		if binary && !*force {
			errhandle(
				errors.New("supply --force to force change of binary file"),
				false, "")
		}
		if !changed {
			changed = true
			colors.Printf("@g%s", fn)
		}

		changenum += 1
		return []byte(*replace)
	})

	if changenum > 0 {
		colors.Printf("@!@y - %d change%s made\n",
			changenum, getSuffix(changenum))
	}

	return changed, result
}

type LineInfo struct {
	num  int
	line []byte
}

// will return slice of [linenum, line] slices
func (v *GRVisitor) FindAllIndex(content []byte) (res []*LineInfo) {
	linenum := 1

	if *singleline {
		begin, end := 0, 0
		for i := 0; i < len(content); i++ {
			if content[i] == '\n' {
				end = i
				line := content[begin:end]
				if v.pattern.Match(line) {
					res = append(res, &LineInfo{linenum, line})
				}
				linenum += 1
				begin = end + 1
			}
		}
		return res
	}

	last := 0
	for _, bounds := range v.pattern.FindAllIndex(content, -1) {
		linenum += bytes.Count(content[last:bounds[0]], byteNewLine)
		last = bounds[0]
		begin, end := beginend(content, bounds[0], bounds[1])
		res = append(res, &LineInfo{linenum, content[begin:end]})
	}
	return res
}

// Given a []byte, start and finish of some inner slice, will find nearest
// newlines on both ends of this slice
func beginend(s []byte, start int, finish int) (begin int, end int) {
	begin = 0
	end = len(s)

	for i := start; i >= 0; i-- {
		if s[i] == byteNewLine[0] {
			begin = i + 1
			break
		}
	}

	// -1 to check if current location is not end of string
	for i := finish - 1; i < len(s); i++ {
		if s[i] == byteNewLine[0] {
			end = i
			break
		}
	}

	return
}

type IntList []int

func (il IntList) Contains(i int) bool {
	for _, x := range il {
		if x == i {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package gosync

import (
	"io/ioutil"
	"os"
	"strings"

	"github.com/mitchellh/goamz/s3"
)

type S3Url struct {
	Url string
}

func (r *S3Url) Bucket() string {
	return r.keys()[0]
}

func (r *S3Url) Key() string {
	return strings.Join(r.keys()[1:len(r.keys())], "/")
}

func (r *S3Url) Path() string {
	return r.Key()
}

func (r *S3Url) Valid() bool {
	return strings.HasPrefix(r.Url, "s3://")
}

func (r *S3Url) keys() []string {
	trimmed_string := strings.TrimLeft(r.Url, "s3://")
	return strings.Split(trimmed_string, "/")
}

func Get(file string, bucket *s3.Bucket, path string) {
	data, err := bucket.Get(path)
	if err != nil {
		panic(err.Error())
	}
	perms := os.FileMode(0644)

	err = ioutil.WriteFile(file, data, perms)
	if err != nil {
		panic(err.Error())
	}
}

func Put(bucket *s3.Bucket, path string, file string) {
	contType := "binary/octet-stream"
	Perms := s3.ACL("private")

	data, err := ioutil.ReadFile(file)
	if err != nil {
		panic(err.Error())
	}

	err = bucket.Put(path, data, contType, Perms)
	if err != nil {
		panic(err.Error())
	}
}
<commit_msg>sniff file mime type and use it<commit_after>package gosync

import (
	"io/ioutil"
	"os"
	"strings"
	"path/filepath"
	"mime"
	"github.com/mitchellh/goamz/s3"
)

type S3Url struct {
	Url string
}

func (r *S3Url) Bucket() string {
	return r.keys()[0]
}

func (r *S3Url) Key() string {
	return strings.Join(r.keys()[1:len(r.keys())], "/")
}

func (r *S3Url) Path() string {
	return r.Key()
}

func (r *S3Url) Valid() bool {
	return strings.HasPrefix(r.Url, "s3://")
}

func (r *S3Url) keys() []string {
	trimmed_string := strings.TrimLeft(r.Url, "s3://")
	return strings.Split(trimmed_string, "/")
}

func Get(file string, bucket *s3.Bucket, path string) {
	data, err := bucket.Get(path)
	if err != nil {
		panic(err.Error())
	}
	perms := os.FileMode(0644)

	err = ioutil.WriteFile(file, data, perms)
	if err != nil {
		panic(err.Error())
	}
}

func Put(bucket *s3.Bucket, path string, file string) {
	contType := mime.TypeByExtension(filepath.Ext(file))
	Perms := s3.ACL("private")

	data, err := ioutil.ReadFile(file)
	if err != nil {
		panic(err.Error())
	}

	err = bucket.Put(path, data, contType, Perms)
	if err != nil {
		panic(err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.11

package ebiten

const __EBITEN_REQUIRES_GO_VERSION_1_11_OR_LATER__ = true
<commit_msg>Add comments (#777)<commit_after>// Copyright 2019 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.11

package ebiten

// Between Go 1.10 and Go 1.11, ioutil.TempFile's behavior is different.
// Ebiten forces the Go version in order to avoid confusion. (#777)

const __EBITEN_REQUIRES_GO_VERSION_1_11_OR_LATER__ = true
<|endoftext|>


<commit_before>package graphite

import (
	"bufio"
	"io"
	"io/ioutil"
	"net/url"
	"time"

	"github.com/ctrlok/tsdbb/interfaces"
)

// Metric is a
type Metric [4][2]byte

// Internal is tuple method for Metric interface
func (m *Metric) Internal() interface{} {
	return ""
}

// PregeneratedMetrics is a struct which contain Metric, and generated by TSDB.GenerateMetrics
type PregeneratedMetrics struct {
	metrics []Metric
	err     error
}

// Metric return metric or out of index error
func (p *PregeneratedMetrics) Metric(i int) (interfaces.Metric, error) {
	if i >= len(p.metrics) {
		return &Metric{}, p.err
	}
	return &p.metrics[i], nil
}

// TSDB is a main generator of graphite Metric and Sender
type TSDB struct {
	GeneratorPrefix []byte
	DevNull         bool
}

// GenerateMetrics is a method for create PregeneratedMetrics
func (t *TSDB) GenerateMetrics(i int) interfaces.PregeneratedMetrics {
	p := PregeneratedMetrics{}
	p.metrics = make([]Metric, i)
	for i := range p.metrics[0] {
		p.metrics[0][i][0] = 48
		p.metrics[0][i][1] = 48
	}
	for n := 1; n < i; n++ {
		var plus byte = 1
		for k := 3; k > -1; k-- {
			for m := 1; m > -1; m-- {
				if plus == 1 {
					if p.metrics[n-1][k][m] == 57 {
						p.metrics[n][k][m] = 48
					} else {
						p.metrics[n][k][m] = p.metrics[n-1][k][m] + plus
						plus = 0
					}
				} else {
					p.metrics[n][k][m] = p.metrics[n-1][k][m]
				}
			}
		}
	}
	return &p
}

// NewSender will create new sender
func (t *TSDB) NewSender(uri *url.URL) (s interfaces.Sender, err error) {
	if t.DevNull {
		return t.newSenderNull(uri)
	}
	return t.newSenderNull(uri)
}
func (t *TSDB) newSenderNull(uri *url.URL) (s interfaces.Sender, err error) {
	var sender SenderNull
	// sender.f, _ = os.OpenFile("/tmp/metricTEst", os.O_RDWR, 0755)
	sender.f = ioutil.Discard
	sender.w = bufio.NewWriter(sender.f)
	return &sender, nil
}

// func (t *TSDB) newSenderUri(uri *url.URL) (s interfaces.Sender, err error) {
// }

// SenderNull is a sender instance.
type SenderNull struct {
	f      io.Writer
	w      *bufio.Writer
	prefix []byte

	host string
}

// Send is a method for sending messages. Work only with internal Metric
func (s *SenderNull) Send(metric interfaces.Metric, t *time.Time) (err error) {
	m := metric.(*Metric)
	_, err = s.w.Write(s.prefix)
	if err != nil {
		return err
	}
	for i := 0; i < 4; i++ {
		err = s.w.WriteByte(46) // dot
		if err != nil {
			return err
		}
		err = s.w.WriteByte(m[i][0])
		if err != nil {
			return err
		}
		err = s.w.WriteByte(m[i][1])
		if err != nil {
			return err
		}
		// s.w.Flush()
	}
	err = s.w.WriteByte(10) // newline
	return err
}

// GetHost will return host of sender
func (s *SenderNull) GetHost() string {
	return s.host
}
<commit_msg>add basic graphite writer implementation<commit_after>package graphite

import (
	"bufio"
	"io"
	"io/ioutil"
	"net"
	"net/url"
	"time"

	"github.com/ctrlok/tsdbb/interfaces"
	"github.com/davecgh/go-spew/spew"
)

// Metric is a
type Metric [4][2]byte

// Internal is tuple method for Metric interface
func (m *Metric) Internal() interface{} {
	return ""
}

// PregeneratedMetrics is a struct which contain Metric, and generated by TSDB.GenerateMetrics
type PregeneratedMetrics struct {
	metrics []Metric
	err     error
}

// Metric return metric or out of index error
func (p *PregeneratedMetrics) Metric(i int) (interfaces.Metric, error) {
	if i >= len(p.metrics) {
		return &Metric{}, p.err
	}
	return &p.metrics[i], nil
}

// TSDB is a main generator of graphite Metric and Sender
type TSDB struct {
	GeneratorPrefix []byte
	DevNull         bool
}

// GenerateMetrics is a method for create PregeneratedMetrics
func (t *TSDB) GenerateMetrics(i int) interfaces.PregeneratedMetrics {
	p := PregeneratedMetrics{}
	p.metrics = make([]Metric, i)
	for i := range p.metrics[0] {
		p.metrics[0][i][0] = 48
		p.metrics[0][i][1] = 48
	}
	for n := 1; n < i; n++ {
		var plus byte = 1
		for k := 3; k > -1; k-- {
			for m := 1; m > -1; m-- {
				if plus == 1 {
					if p.metrics[n-1][k][m] == 57 {
						p.metrics[n][k][m] = 48
					} else {
						p.metrics[n][k][m] = p.metrics[n-1][k][m] + plus
						plus = 0
					}
				} else {
					p.metrics[n][k][m] = p.metrics[n-1][k][m]
				}
			}
		}
	}
	return &p
}

// NewSender will create new sender
func (t *TSDB) NewSender(uri *url.URL) (s interfaces.Sender, err error) {
	sender := Sender{}
	if t.DevNull {
		// sender.f, _ = os.OpenFile("/tmp/metricTEst", os.O_RDWR, 0755)
		sender.f = ioutil.Discard
		sender.w = bufio.NewWriter(sender.f)
		return &sender, nil
	}
	addr, err := net.ResolveTCPAddr(uri.Scheme, uri.Host)
	if err != nil {
		return nil, err
	}
	conn, err := net.DialTCP(uri.Scheme, nil, addr)
	spew.Dump(conn)
	if err != nil {
		return nil, err
	}
	sender.f = conn
	sender.w = bufio.NewWriter(sender.f)
	return &sender, err
}

// Sender is a sender instance.
type Sender struct {
	w      *bufio.Writer
	f      io.Writer
	prefix []byte

	host string
}

// Send is a method for sending messages. Work only with internal Metric
func (s *Sender) Send(metric interfaces.Metric, t *time.Time) (err error) {
	m := metric.(*Metric)
	_, err = s.w.Write(s.prefix)
	if err != nil {
		return err
	}
	for i := 0; i < 4; i++ {
		err = s.w.WriteByte(46) // dot
		if err != nil {
			return err
		}
		err = s.w.WriteByte(m[i][0])
		if err != nil {
			return err
		}
		err = s.w.WriteByte(m[i][1])
		if err != nil {
			return err
		}
		// s.w.Flush()
	}
	err = s.w.WriteByte(10) // newline
	return err
}

// GetHost will return host of sender
func (s *Sender) GetHost() string {
	return s.host
}
<|endoftext|>


<commit_before>package slogger

import "github.com/tychoish/grip/level"

// Level represents slogger's level types. In the original
// implementation there are four levels and an "OFF" value.
type Level uint8

// slogger has its own system of priorities/log levels. These
// constants represent those levels, and the Level type can be
// converted to grip level.Priority values.
const (
	OFF Level = iota
	DEBUG
	INFO
	WARN
	ERROR
)

func (l Level) String() string {
	switch l {
	case OFF:
		return "off"
	case DEBUG:
		return "debug"
	case INFO:
		return "info"
	case WARN:
		return "warn"
	default:
		return ""
	}
}

// Priority returns grip's native level.Priority for a slogger.Level.
func (l Level) Priority() level.Priority {
	switch l {
	case OFF:
		return level.Invalid
	case DEBUG:
		return level.Debug
	case INFO:
		return level.Info
	case WARN:
		return level.Warning
	default:
		return level.Notice
	}
}

func convertFromPriority(l level.Priority) Level {
	switch l {
	case level.Emergency, level.Alert, level.Critical, level.Error, level.Warning:
		return WARN
	case level.Notice, level.Info:
		return INFO
	case level.Debug:
		return DEBUG
	case level.Invalid:
		return OFF
	default:
		return INFO
	}
}
<commit_msg>add error level to slogger interface<commit_after>package slogger

import "github.com/tychoish/grip/level"

// Level represents slogger's level types. In the original
// implementation there are four levels and an "OFF" value.
type Level uint8

// slogger has its own system of priorities/log levels. These
// constants represent those levels, and the Level type can be
// converted to grip level.Priority values.
const (
	OFF Level = iota
	DEBUG
	INFO
	WARN
	ERROR
)

func (l Level) String() string {
	switch l {
	case OFF:
		return "off"
	case DEBUG:
		return "debug"
	case INFO:
		return "info"
	case WARN:
		return "warn"
	case ERROR:
		return "error"
	default:
		return ""
	}
}

// Priority returns grip's native level.Priority for a slogger.Level.
func (l Level) Priority() level.Priority {
	switch l {
	case OFF:
		return level.Invalid
	case DEBUG:
		return level.Debug
	case INFO:
		return level.Info
	case WARN:
		return level.Warning
	case ERROR:
		return level.Error
	default:
		return level.Notice
	}
}

func convertFromPriority(l level.Priority) Level {
	switch l {
	case level.Emergency, level.Alert, level.Critical, level.Error:
		return ERROR
	case level.Warning:
		return WARN
	case level.Notice, level.Info:
		return INFO
	case level.Debug:
		return DEBUG
	case level.Invalid:
		return OFF
	default:
		return INFO
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/ian-kent/go-log/log"
	"github.com/ian-kent/gopan/gopan"
	gotcha "github.com/ian-kent/gotcha/app"
	"github.com/ian-kent/gotcha/events"
	"github.com/ian-kent/gotcha/http"
	"html/template"
	"io/ioutil"
	nethttp "net/http"
	"strings"
	"sync"
)

var CurrentRelease = "0.3b"

type Releases []*Release
type Release struct {
	TagName string `json:"tag_name"`
	URL     string `json:"html_url"`
}

func main() {
	configure()

	config.CurrentRelease = CurrentRelease

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()

		// Load our secondary indexes
		indexes = make(map[string]map[string]*gopan.Source)
		for _, idx := range config.Indexes {
			indexes[idx] = gopan.LoadIndex(config.CacheDir + "/" + idx)
		}

		// Load our primary index (this is the only index written back to)
		indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
	}()

	go func() {
		wg.Wait()
		wg.Add(1)
		go func() {
			wg.Wait()
			config.ImportAvailable = true

			nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
			// TODO should probably be in the index - needs to udpate when index changes
			summary = &Summary{nsrc, nauth, npkg, nprov}

			// Do this now so changing the level doesn't interfere with index load
			log.Logger().SetLevel(log.Stol(config.LogLevel))
		}()
		defer wg.Done()
		// Create in-memory indexes for UI/search etc
		for fname, _ := range indexes {
			for idn, idx := range indexes[fname] {
				mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
				for _, auth := range idx.Authors {
					// author name
					if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
						mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
						mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth

					// wildcards
					if _, ok := mapped[idx.Name]["*"]; !ok {
						mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
						mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name]["*"]["**"][auth.Name] = auth

					// combos
					if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
						mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
						mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
					mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth

					for _, pkg := range auth.Packages {
						filemap[pkg.AuthorURL()] = idn
						for _, prov := range pkg.Provides {
							parts := strings.Split(prov.Name, "::")
							log.Trace("PACKAGE: %s", prov.Name)

							if _, ok := packages[parts[0]]; !ok {
								packages[parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if _, ok := idxpackages[idx.Name]; !ok {
								idxpackages[idx.Name] = make(map[string]*PkgSpace)
							}
							if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
								idxpackages[idx.Name][parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if len(parts) == 1 {
								packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
								packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
								idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
							} else {
								packages[parts[0]].Populate(parts[1:], prov)
								idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
							}
						}
					}
				}
			}
		}
	}()

	// Get latest SmartPAN version
	go func() {
		res, err := nethttp.Get("https://api.github.com/repos/ian-kent/gopan/releases")
		if err != nil {
			log.Error("Error getting latest version: %s", err.Error())
			return
		}
		defer res.Body.Close()
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Error("Error reading stream: %s", err.Error())
			return
		}

		var r Releases
		err = json.Unmarshal(b, &r)
		if err != nil {
			log.Error("Error unmarshalling JSON: %s", err.Error())
			return
		}

		log.Info("Current release: %s", config.CurrentRelease)
		rel := strings.TrimPrefix(r[0].TagName, "v")
		log.Info("Latest release: %s", rel)
		config.LatestRelease = rel
		config.UpdateURL = r[0].URL

		if config.CurrentRelease < rel {
			config.CanUpdate = true
			log.Info("Your version of SmartPAN can be updated.")
		}
	}()

	// Create our Gotcha application
	var app = gotcha.Create(Asset)
	app.Config.Listen = config.Bind

	summary = &Summary{0, 0, 0, 0}

	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		session.Stash["summary"] = summary
		session.Stash["config"] = config

		next()
	})

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", search)
	r.Post("/", search)

	r.Get("/help", help)
	r.Get("/settings", settings)
	r.Get("/browse", browse)

	r.Get("/import", import1)
	r.Post("/import", import1)

	r.Get("/import/(?P<jobid>[^/]+)", import2)
	r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// JSON endpoints
	r.Get("/where/(?P<module>[^/]+)/?", where)
	r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)

	// Put these last so they only match /{repo} if nothing else matches
	r.Get("/(?P<repo>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
	r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
	r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)

	// Start our application
	app.Start()

	<-make(chan int)
}

func help(session *http.Session) {
	session.Stash["Title"] = "SmartPAN Help"
	html, _ := session.RenderTemplate("help.html")

	session.Stash["Page"] = "Help"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}

func settings(session *http.Session) {
	session.Stash["Title"] = "SmartPAN Settings"
	html, _ := session.RenderTemplate("settings.html")

	session.Stash["Page"] = "Settings"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}
<commit_msg>Update version number<commit_after>package main

import (
	"encoding/json"
	"github.com/ian-kent/go-log/log"
	"github.com/ian-kent/gopan/gopan"
	gotcha "github.com/ian-kent/gotcha/app"
	"github.com/ian-kent/gotcha/events"
	"github.com/ian-kent/gotcha/http"
	"html/template"
	"io/ioutil"
	nethttp "net/http"
	"strings"
	"sync"
)

var CurrentRelease = "0.3c"

type Releases []*Release
type Release struct {
	TagName string `json:"tag_name"`
	URL     string `json:"html_url"`
}

func main() {
	configure()

	config.CurrentRelease = CurrentRelease

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()

		// Load our secondary indexes
		indexes = make(map[string]map[string]*gopan.Source)
		for _, idx := range config.Indexes {
			indexes[idx] = gopan.LoadIndex(config.CacheDir + "/" + idx)
		}

		// Load our primary index (this is the only index written back to)
		indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
	}()

	go func() {
		wg.Wait()
		wg.Add(1)
		go func() {
			wg.Wait()
			config.ImportAvailable = true

			nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
			// TODO should probably be in the index - needs to udpate when index changes
			summary = &Summary{nsrc, nauth, npkg, nprov}

			// Do this now so changing the level doesn't interfere with index load
			log.Logger().SetLevel(log.Stol(config.LogLevel))
		}()
		defer wg.Done()
		// Create in-memory indexes for UI/search etc
		for fname, _ := range indexes {
			for idn, idx := range indexes[fname] {
				mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
				for _, auth := range idx.Authors {
					// author name
					if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
						mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
						mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth

					// wildcards
					if _, ok := mapped[idx.Name]["*"]; !ok {
						mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
						mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name]["*"]["**"][auth.Name] = auth

					// combos
					if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
						mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
						mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
					mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth

					for _, pkg := range auth.Packages {
						filemap[pkg.AuthorURL()] = idn
						for _, prov := range pkg.Provides {
							parts := strings.Split(prov.Name, "::")
							log.Trace("PACKAGE: %s", prov.Name)

							if _, ok := packages[parts[0]]; !ok {
								packages[parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if _, ok := idxpackages[idx.Name]; !ok {
								idxpackages[idx.Name] = make(map[string]*PkgSpace)
							}
							if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
								idxpackages[idx.Name][parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if len(parts) == 1 {
								packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
								packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
								idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
							} else {
								packages[parts[0]].Populate(parts[1:], prov)
								idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
							}
						}
					}
				}
			}
		}
	}()

	// Get latest SmartPAN version
	go func() {
		res, err := nethttp.Get("https://api.github.com/repos/ian-kent/gopan/releases")
		if err != nil {
			log.Error("Error getting latest version: %s", err.Error())
			return
		}
		defer res.Body.Close()
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Error("Error reading stream: %s", err.Error())
			return
		}

		var r Releases
		err = json.Unmarshal(b, &r)
		if err != nil {
			log.Error("Error unmarshalling JSON: %s", err.Error())
			return
		}

		log.Info("Current release: %s", config.CurrentRelease)
		rel := strings.TrimPrefix(r[0].TagName, "v")
		log.Info("Latest release: %s", rel)
		config.LatestRelease = rel
		config.UpdateURL = r[0].URL

		if config.CurrentRelease < rel {
			config.CanUpdate = true
			log.Info("Your version of SmartPAN can be updated.")
		}
	}()

	// Create our Gotcha application
	var app = gotcha.Create(Asset)
	app.Config.Listen = config.Bind

	summary = &Summary{0, 0, 0, 0}

	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		session.Stash["summary"] = summary
		session.Stash["config"] = config

		next()
	})

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", search)
	r.Post("/", search)

	r.Get("/help", help)
	r.Get("/settings", settings)
	r.Get("/browse", browse)

	r.Get("/import", import1)
	r.Post("/import", import1)

	r.Get("/import/(?P<jobid>[^/]+)", import2)
	r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// JSON endpoints
	r.Get("/where/(?P<module>[^/]+)/?", where)
	r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)

	// Put these last so they only match /{repo} if nothing else matches
	r.Get("/(?P<repo>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
	r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
	r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)

	// Start our application
	app.Start()

	<-make(chan int)
}

func help(session *http.Session) {
	session.Stash["Title"] = "SmartPAN Help"
	html, _ := session.RenderTemplate("help.html")

	session.Stash["Page"] = "Help"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}

func settings(session *http.Session) {
	session.Stash["Title"] = "SmartPAN Settings"
	html, _ := session.RenderTemplate("settings.html")

	session.Stash["Page"] = "Settings"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}
<|endoftext|>


<commit_before>package logger

import (
	"bytes"
	"log"
	"net/http"

	"go.uber.org/zap"
)

var Log = setUpDefaultLogger().Sugar()

func setUpDefaultLogger() *zap.Logger {
	logger, e := zap.NewDevelopment()
	if e != nil {
		panic(e)
	}
	return logger
}

func SetLogger(logger *zap.Logger) {
	Log = logger.Sugar()
}

func From(r *http.Request) *zap.SugaredLogger {
	log := r.Context().Value("logger")
	if log != nil {
		return log.(*zap.SugaredLogger)
	} else {
		return Log
	}
}

// Copied from go.uber.org/zap/global.go and changed to use Error instead of Info:
func NewStdLog(l *zap.Logger) *log.Logger {
	const (
		_stdLogDefaultDepth = 2
		_loggerWriterDepth  = 1
	)
	return log.New(&loggerWriter{l.WithOptions(
		zap.AddCallerSkip(_stdLogDefaultDepth + _loggerWriterDepth),
	)}, "" /* prefix */, 0 /* flags */)
}

type loggerWriter struct{ logger *zap.Logger }

func (l *loggerWriter) Write(p []byte) (int, error) {
	p = bytes.TrimSpace(p)
	l.logger.Error(string(p))
	return len(p), nil
}
<commit_msg>Disable stacktraces when using the default logger<commit_after>package logger

import (
	"bytes"
	"log"
	"net/http"

	"go.uber.org/zap"
)

var Log = setUpDefaultLogger().Sugar()

func setUpDefaultLogger() *zap.Logger {
	cfg := zap.NewDevelopmentConfig()
	cfg.DisableStacktrace = true
	logger, e := cfg.Build()
	if e != nil {
		panic(e)
	}
	return logger
}

func SetLogger(logger *zap.Logger) {
	Log = logger.Sugar()
}

func From(r *http.Request) *zap.SugaredLogger {
	log := r.Context().Value("logger")
	if log != nil {
		return log.(*zap.SugaredLogger)
	} else {
		return Log
	}
}

// Copied from go.uber.org/zap/global.go and changed to use Error instead of Info:
func NewStdLog(l *zap.Logger) *log.Logger {
	const (
		_stdLogDefaultDepth = 2
		_loggerWriterDepth  = 1
	)
	return log.New(&loggerWriter{l.WithOptions(
		zap.AddCallerSkip(_stdLogDefaultDepth + _loggerWriterDepth),
	)}, "" /* prefix */, 0 /* flags */)
}

type loggerWriter struct{ logger *zap.Logger }

func (l *loggerWriter) Write(p []byte) (int, error) {
	p = bytes.TrimSpace(p)
	l.logger.Error(string(p))
	return len(p), nil
}
<|endoftext|>


<commit_before>package datastore_test

import (
	"bytes"
	"math/rand"
	"path"
	"strings"
	"testing"

	. "github.com/ipfs/go-datastore"
	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

func randomString() string {
	chars := "abcdefghijklmnopqrstuvwxyz1234567890"
	var buf bytes.Buffer
	l := rand.Intn(50)
	for j := 0; j < l; j++ {
		buf.WriteByte(chars[rand.Intn(len(chars))])
	}
	return buf.String()
}

type KeySuite struct{}

var _ = Suite(&KeySuite{})

func (ks *KeySuite) SubtestKey(s string, c *C) {
	fixed := path.Clean("/" + s)
	namespaces := strings.Split(fixed, "/")[1:]
	lastNamespace := namespaces[len(namespaces)-1]
	lnparts := strings.Split(lastNamespace, ":")
	ktype := ""
	if len(lnparts) > 1 {
		ktype = strings.Join(lnparts[:len(lnparts)-1], ":")
	}
	kname := lnparts[len(lnparts)-1]

	kchild := path.Clean(fixed + "/cchildd")
	kparent := "/" + strings.Join(append(namespaces[:len(namespaces)-1]), "/")
	kpath := path.Clean(kparent + "/" + ktype)
	kinstance := fixed + ":" + "inst"

	c.Log("Testing: ", NewKey(s))

	c.Check(NewKey(s).String(), Equals, fixed)
	c.Check(NewKey(s), Equals, NewKey(s))
	c.Check(NewKey(s).String(), Equals, NewKey(s).String())
	c.Check(NewKey(s).Name(), Equals, kname)
	c.Check(NewKey(s).Type(), Equals, ktype)
	c.Check(NewKey(s).Path().String(), Equals, kpath)
	c.Check(NewKey(s).Instance("inst").String(), Equals, kinstance)

	c.Check(NewKey(s).Child(NewKey("cchildd")).String(), Equals, kchild)
	c.Check(NewKey(s).Child(NewKey("cchildd")).Parent().String(), Equals, fixed)
	c.Check(NewKey(s).ChildString("cchildd").String(), Equals, kchild)
	c.Check(NewKey(s).ChildString("cchildd").Parent().String(), Equals, fixed)
	c.Check(NewKey(s).Parent().String(), Equals, kparent)
	c.Check(len(NewKey(s).List()), Equals, len(namespaces))
	c.Check(len(NewKey(s).Namespaces()), Equals, len(namespaces))
	for i, e := range NewKey(s).List() {
		c.Check(namespaces[i], Equals, e)
	}

	c.Check(NewKey(s), Equals, NewKey(s))
	c.Check(NewKey(s).Equal(NewKey(s)), Equals, true)
	c.Check(NewKey(s).Equal(NewKey("/fdsafdsa/"+s)), Equals, false)

	// less
	c.Check(NewKey(s).Less(NewKey(s).Parent()), Equals, false)
	c.Check(NewKey(s).Less(NewKey(s).ChildString("foo")), Equals, true)
}

func (ks *KeySuite) TestKeyBasic(c *C) {
	ks.SubtestKey("", c)
	ks.SubtestKey("abcde", c)
	ks.SubtestKey("disahfidsalfhduisaufidsail", c)
	ks.SubtestKey("/fdisahfodisa/fdsa/fdsafdsafdsafdsa/fdsafdsa/", c)
	ks.SubtestKey("4215432143214321432143214321", c)
	ks.SubtestKey("/fdisaha////fdsa////fdsafdsafdsafdsa/fdsafdsa/", c)
	ks.SubtestKey("abcde:fdsfd", c)
	ks.SubtestKey("disahfidsalfhduisaufidsail:fdsa", c)
	ks.SubtestKey("/fdisahfodisa/fdsa/fdsafdsafdsafdsa/fdsafdsa/:", c)
	ks.SubtestKey("4215432143214321432143214321:", c)
	ks.SubtestKey("fdisaha////fdsa////fdsafdsafdsafdsa/fdsafdsa/f:fdaf", c)
}

func CheckTrue(c *C, cond bool) {
	c.Check(cond, Equals, true)
}

func (ks *KeySuite) TestKeyAncestry(c *C) {
	k1 := NewKey("/A/B/C")
	k2 := NewKey("/A/B/C/D")

	c.Check(k1.String(), Equals, "/A/B/C")
	c.Check(k2.String(), Equals, "/A/B/C/D")
	CheckTrue(c, k1.IsAncestorOf(k2))
	CheckTrue(c, k2.IsDescendantOf(k1))
	CheckTrue(c, NewKey("/A").IsAncestorOf(k2))
	CheckTrue(c, NewKey("/A").IsAncestorOf(k1))
	CheckTrue(c, !NewKey("/A").IsDescendantOf(k2))
	CheckTrue(c, !NewKey("/A").IsDescendantOf(k1))
	CheckTrue(c, k2.IsDescendantOf(NewKey("/A")))
	CheckTrue(c, k1.IsDescendantOf(NewKey("/A")))
	CheckTrue(c, !k2.IsAncestorOf(NewKey("/A")))
	CheckTrue(c, !k1.IsAncestorOf(NewKey("/A")))
	CheckTrue(c, !k2.IsAncestorOf(k2))
	CheckTrue(c, !k1.IsAncestorOf(k1))
	c.Check(k1.Child(NewKey("D")).String(), Equals, k2.String())
	c.Check(k1.ChildString("D").String(), Equals, k2.String())
	c.Check(k1.String(), Equals, k2.Parent().String())
	c.Check(k1.Path().String(), Equals, k2.Parent().Path().String())
}

func (ks *KeySuite) TestType(c *C) {
	k1 := NewKey("/A/B/C:c")
	k2 := NewKey("/A/B/C:c/D:d")

	CheckTrue(c, k1.IsAncestorOf(k2))
	CheckTrue(c, k2.IsDescendantOf(k1))
	c.Check(k1.Type(), Equals, "C")
	c.Check(k2.Type(), Equals, "D")
	c.Check(k1.Type(), Equals, k2.Parent().Type())
}

func (ks *KeySuite) TestRandom(c *C) {
	keys := map[Key]bool{}
	for i := 0; i < 1000; i++ {
		r := RandomKey()
		_, found := keys[r]
		CheckTrue(c, !found)
		keys[r] = true
	}
	CheckTrue(c, len(keys) == 1000)
}

func (ks *KeySuite) TestLess(c *C) {

	checkLess := func(a, b string) {
		ak := NewKey(a)
		bk := NewKey(b)
		c.Check(ak.Less(bk), Equals, true)
		c.Check(bk.Less(ak), Equals, false)
	}

	checkLess("/a/b/c", "/a/b/c/d")
	checkLess("/a/b", "/a/b/c/d")
	checkLess("/a", "/a/b/c/d")
	checkLess("/a/a/c", "/a/b/c")
	checkLess("/a/a/d", "/a/b/c")
	checkLess("/a/b/c/d/e/f/g/h", "/b")
	checkLess("/", "/a")
}

func TestKeyMarshalJSON(t *testing.T) {
	cases := []struct {
		key  Key
		data []byte
		err  string
	}{
		{NewKey("/a/b/c"), []byte("\"/a/b/c\""), ""},
		{NewKey("/shouldescapekey\"/with/quote"), []byte("\"/shouldescapekey\\\"/with/quote\""), ""},
	}

	for i, c := range cases {
		out, err := c.key.MarshalJSON()
		if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) {
			t.Errorf("case %d marshal error mismatch: expected: %s, got: %s", i, c.err, err)
		}
		if !bytes.Equal(c.data, out) {
			t.Errorf("case %d value mismatch: expected: %s, got: %s", i, string(c.data), string(out))
		}

		if c.err == "" {
			key := Key{}
			if err := key.UnmarshalJSON(out); err != nil {
				t.Errorf("case %d error parsing key from json output: %s", i, err.Error())
			}
			if !c.key.Equal(key) {
				t.Errorf("case %d parsed key from json output mismatch. expected: %s, got: %s", i, c.key.String(), key.String())
			}
		}
	}
}

func TestKeyUnmarshalJSON(t *testing.T) {
	cases := []struct {
		data []byte
		key  Key
		err  string
	}{
		{[]byte("\"/a/b/c\""), NewKey("/a/b/c"), ""},
		{[]byte{}, Key{}, "unexpected end of JSON input"},
		{[]byte{'"'}, Key{}, "unexpected end of JSON input"},
		{[]byte(`""`), NewKey(""), ""},
	}

	for i, c := range cases {
		key := Key{}
		err := key.UnmarshalJSON(c.data)
		if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) {
			t.Errorf("case %d marshal error mismatch: expected: %s, got: %s", i, c.err, err)
		}

		if !key.Equal(c.key) {
			t.Errorf("case %d key mismatch: expected: %s, got: %s", i, c.key, key)
		}
	}
}
<commit_msg>fix gx tests<commit_after>package datastore_test

import (
	"bytes"
	"math/rand"
	"path"
	"strings"
	"testing"

	. "github.com/go-check/check"
	. "github.com/ipfs/go-datastore"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

func randomString() string {
	chars := "abcdefghijklmnopqrstuvwxyz1234567890"
	var buf bytes.Buffer
	l := rand.Intn(50)
	for j := 0; j < l; j++ {
		buf.WriteByte(chars[rand.Intn(len(chars))])
	}
	return buf.String()
}

type KeySuite struct{}

var _ = Suite(&KeySuite{})

func (ks *KeySuite) SubtestKey(s string, c *C) {
	fixed := path.Clean("/" + s)
	namespaces := strings.Split(fixed, "/")[1:]
	lastNamespace := namespaces[len(namespaces)-1]
	lnparts := strings.Split(lastNamespace, ":")
	ktype := ""
	if len(lnparts) > 1 {
		ktype = strings.Join(lnparts[:len(lnparts)-1], ":")
	}
	kname := lnparts[len(lnparts)-1]

	kchild := path.Clean(fixed + "/cchildd")
	kparent := "/" + strings.Join(append(namespaces[:len(namespaces)-1]), "/")
	kpath := path.Clean(kparent + "/" + ktype)
	kinstance := fixed + ":" + "inst"

	c.Log("Testing: ", NewKey(s))

	c.Check(NewKey(s).String(), Equals, fixed)
	c.Check(NewKey(s), Equals, NewKey(s))
	c.Check(NewKey(s).String(), Equals, NewKey(s).String())
	c.Check(NewKey(s).Name(), Equals, kname)
	c.Check(NewKey(s).Type(), Equals, ktype)
	c.Check(NewKey(s).Path().String(), Equals, kpath)
	c.Check(NewKey(s).Instance("inst").String(), Equals, kinstance)

	c.Check(NewKey(s).Child(NewKey("cchildd")).String(), Equals, kchild)
	c.Check(NewKey(s).Child(NewKey("cchildd")).Parent().String(), Equals, fixed)
	c.Check(NewKey(s).ChildString("cchildd").String(), Equals, kchild)
	c.Check(NewKey(s).ChildString("cchildd").Parent().String(), Equals, fixed)
	c.Check(NewKey(s).Parent().String(), Equals, kparent)
	c.Check(len(NewKey(s).List()), Equals, len(namespaces))
	c.Check(len(NewKey(s).Namespaces()), Equals, len(namespaces))
	for i, e := range NewKey(s).List() {
		c.Check(namespaces[i], Equals, e)
	}

	c.Check(NewKey(s), Equals, NewKey(s))
	c.Check(NewKey(s).Equal(NewKey(s)), Equals, true)
	c.Check(NewKey(s).Equal(NewKey("/fdsafdsa/"+s)), Equals, false)

	// less
	c.Check(NewKey(s).Less(NewKey(s).Parent()), Equals, false)
	c.Check(NewKey(s).Less(NewKey(s).ChildString("foo")), Equals, true)
}

func (ks *KeySuite) TestKeyBasic(c *C) {
	ks.SubtestKey("", c)
	ks.SubtestKey("abcde", c)
	ks.SubtestKey("disahfidsalfhduisaufidsail", c)
	ks.SubtestKey("/fdisahfodisa/fdsa/fdsafdsafdsafdsa/fdsafdsa/", c)
	ks.SubtestKey("4215432143214321432143214321", c)
	ks.SubtestKey("/fdisaha////fdsa////fdsafdsafdsafdsa/fdsafdsa/", c)
	ks.SubtestKey("abcde:fdsfd", c)
	ks.SubtestKey("disahfidsalfhduisaufidsail:fdsa", c)
	ks.SubtestKey("/fdisahfodisa/fdsa/fdsafdsafdsafdsa/fdsafdsa/:", c)
	ks.SubtestKey("4215432143214321432143214321:", c)
	ks.SubtestKey("fdisaha////fdsa////fdsafdsafdsafdsa/fdsafdsa/f:fdaf", c)
}

func CheckTrue(c *C, cond bool) {
	c.Check(cond, Equals, true)
}

func (ks *KeySuite) TestKeyAncestry(c *C) {
	k1 := NewKey("/A/B/C")
	k2 := NewKey("/A/B/C/D")

	c.Check(k1.String(), Equals, "/A/B/C")
	c.Check(k2.String(), Equals, "/A/B/C/D")
	CheckTrue(c, k1.IsAncestorOf(k2))
	CheckTrue(c, k2.IsDescendantOf(k1))
	CheckTrue(c, NewKey("/A").IsAncestorOf(k2))
	CheckTrue(c, NewKey("/A").IsAncestorOf(k1))
	CheckTrue(c, !NewKey("/A").IsDescendantOf(k2))
	CheckTrue(c, !NewKey("/A").IsDescendantOf(k1))
	CheckTrue(c, k2.IsDescendantOf(NewKey("/A")))
	CheckTrue(c, k1.IsDescendantOf(NewKey("/A")))
	CheckTrue(c, !k2.IsAncestorOf(NewKey("/A")))
	CheckTrue(c, !k1.IsAncestorOf(NewKey("/A")))
	CheckTrue(c, !k2.IsAncestorOf(k2))
	CheckTrue(c, !k1.IsAncestorOf(k1))
	c.Check(k1.Child(NewKey("D")).String(), Equals, k2.String())
	c.Check(k1.ChildString("D").String(), Equals, k2.String())
	c.Check(k1.String(), Equals, k2.Parent().String())
	c.Check(k1.Path().String(), Equals, k2.Parent().Path().String())
}

func (ks *KeySuite) TestType(c *C) {
	k1 := NewKey("/A/B/C:c")
	k2 := NewKey("/A/B/C:c/D:d")

	CheckTrue(c, k1.IsAncestorOf(k2))
	CheckTrue(c, k2.IsDescendantOf(k1))
	c.Check(k1.Type(), Equals, "C")
	c.Check(k2.Type(), Equals, "D")
	c.Check(k1.Type(), Equals, k2.Parent().Type())
}

func (ks *KeySuite) TestRandom(c *C) {
	keys := map[Key]bool{}
	for i := 0; i < 1000; i++ {
		r := RandomKey()
		_, found := keys[r]
		CheckTrue(c, !found)
		keys[r] = true
	}
	CheckTrue(c, len(keys) == 1000)
}

func (ks *KeySuite) TestLess(c *C) {

	checkLess := func(a, b string) {
		ak := NewKey(a)
		bk := NewKey(b)
		c.Check(ak.Less(bk), Equals, true)
		c.Check(bk.Less(ak), Equals, false)
	}

	checkLess("/a/b/c", "/a/b/c/d")
	checkLess("/a/b", "/a/b/c/d")
	checkLess("/a", "/a/b/c/d")
	checkLess("/a/a/c", "/a/b/c")
	checkLess("/a/a/d", "/a/b/c")
	checkLess("/a/b/c/d/e/f/g/h", "/b")
	checkLess("/", "/a")
}

func TestKeyMarshalJSON(t *testing.T) {
	cases := []struct {
		key  Key
		data []byte
		err  string
	}{
		{NewKey("/a/b/c"), []byte("\"/a/b/c\""), ""},
		{NewKey("/shouldescapekey\"/with/quote"), []byte("\"/shouldescapekey\\\"/with/quote\""), ""},
	}

	for i, c := range cases {
		out, err := c.key.MarshalJSON()
		if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) {
			t.Errorf("case %d marshal error mismatch: expected: %s, got: %s", i, c.err, err)
		}
		if !bytes.Equal(c.data, out) {
			t.Errorf("case %d value mismatch: expected: %s, got: %s", i, string(c.data), string(out))
		}

		if c.err == "" {
			key := Key{}
			if err := key.UnmarshalJSON(out); err != nil {
				t.Errorf("case %d error parsing key from json output: %s", i, err.Error())
			}
			if !c.key.Equal(key) {
				t.Errorf("case %d parsed key from json output mismatch. expected: %s, got: %s", i, c.key.String(), key.String())
			}
		}
	}
}

func TestKeyUnmarshalJSON(t *testing.T) {
	cases := []struct {
		data []byte
		key  Key
		err  string
	}{
		{[]byte("\"/a/b/c\""), NewKey("/a/b/c"), ""},
		{[]byte{}, Key{}, "unexpected end of JSON input"},
		{[]byte{'"'}, Key{}, "unexpected end of JSON input"},
		{[]byte(`""`), NewKey(""), ""},
	}

	for i, c := range cases {
		key := Key{}
		err := key.UnmarshalJSON(c.data)
		if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) {
			t.Errorf("case %d marshal error mismatch: expected: %s, got: %s", i, c.err, err)
		}

		if !key.Equal(c.key) {
			t.Errorf("case %d key mismatch: expected: %s, got: %s", i, c.key, key)
		}
	}
}
<|endoftext|>


<commit_before>package dht

// get_peers and announce_peers.

import (
	"log"
	"time"

	"github.com/anacrolix/missinggo"
	"github.com/anacrolix/sync"
	"github.com/willf/bloom"

	"github.com/anacrolix/torrent/logonce"
)

// Maintains state for an ongoing Announce operation. An Announce is started
// by calling Server.Announce.
type Announce struct {
	mu    sync.Mutex
	Peers chan PeersValues
	// Inner chan is set to nil when on close.
	values              chan PeersValues
	stop                chan struct{}
	triedAddrs          *bloom.BloomFilter
	pending             int
	server              *Server
	infoHash            string
	numContacted        int
	announcePort        int
	announcePortImplied bool
}

// Returns the number of distinct remote addresses the announce has queried.
func (me *Announce) NumContacted() int {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me.numContacted
}

// This is kind of the main thing you want to do with DHT. It traverses the
// graph toward nodes that store peers for the infohash, streaming them to the
// caller, and announcing the local node to each node if allowed and
// specified.
func (s *Server) Announce(infoHash string, port int, impliedPort bool) (*Announce, error) {
	s.mu.Lock()
	startAddrs := func() (ret []dHTAddr) {
		for _, n := range s.closestGoodNodes(160, infoHash) {
			ret = append(ret, n.addr)
		}
		return
	}()
	s.mu.Unlock()
	if len(startAddrs) == 0 {
		addrs, err := bootstrapAddrs(s.bootstrapNodes)
		if err != nil {
			return nil, err
		}
		for _, addr := range addrs {
			startAddrs = append(startAddrs, newDHTAddr(addr))
		}
	}
	disc := &Announce{
		Peers:               make(chan PeersValues, 100),
		stop:                make(chan struct{}),
		values:              make(chan PeersValues),
		triedAddrs:          bloom.NewWithEstimates(1000, 0.5),
		server:              s,
		infoHash:            infoHash,
		announcePort:        port,
		announcePortImplied: impliedPort,
	}
	// Function ferries from values to Values until discovery is halted.
	go func() {
		defer close(disc.Peers)
		for {
			select {
			case psv := <-disc.values:
				select {
				case disc.Peers <- psv:
				case <-disc.stop:
					return
				}
			case <-disc.stop:
				return
			}
		}
	}()
	for i, addr := range startAddrs {
		if i != 0 {
			time.Sleep(time.Millisecond)
		}
		disc.mu.Lock()
		disc.contact(addr)
		disc.mu.Unlock()
	}
	return disc, nil
}

func (me *Announce) gotNodeAddr(addr dHTAddr) {
	if missinggo.AddrPort(addr) == 0 {
		// Not a contactable address.
		return
	}
	if me.triedAddrs.Test([]byte(addr.String())) {
		return
	}
	if me.server.ipBlocked(addr.UDPAddr().IP) {
		return
	}
	me.server.mu.Lock()
	if me.server.badNodes.Test([]byte(addr.String())) {
		me.server.mu.Unlock()
		return
	}
	me.server.mu.Unlock()
	me.contact(addr)
}

func (me *Announce) contact(addr dHTAddr) {
	me.numContacted++
	me.triedAddrs.Add([]byte(addr.String()))
	if err := me.getPeers(addr); err != nil {
		log.Printf("error sending get_peers request to %s: %#v", addr, err)
		return
	}
	me.pending++
}

func (me *Announce) transactionClosed() {
	me.pending--
	if me.pending == 0 {
		me.close()
		return
	}
}

func (me *Announce) responseNode(node NodeInfo) {
	me.gotNodeAddr(node.Addr)
}

func (me *Announce) closingCh() chan struct{} {
	return me.stop
}

// Announce to a peer, if appropriate.
func (me *Announce) maybeAnnouncePeer(to dHTAddr, token, peerId string) {
	me.server.mu.Lock()
	defer me.server.mu.Unlock()
	if !me.server.config.NoSecurity {
		if len(peerId) != 20 {
			return
		}
		if !NodeIdSecure(peerId, to.IP()) {
			return
		}
	}
	err := me.server.announcePeer(to, me.infoHash, me.announcePort, token, me.announcePortImplied)
	me.server.mu.Unlock()
	if err != nil {
		logonce.Stderr.Printf("error announcing peer: %s", err)
	}
}

func (me *Announce) getPeers(addr dHTAddr) error {
	me.server.mu.Lock()
	defer me.server.mu.Unlock()
	t, err := me.server.getPeers(addr, me.infoHash)
	if err != nil {
		return err
	}
	t.SetResponseHandler(func(m Msg, ok bool) {
		// Register suggested nodes closer to the target info-hash.
		if m.R != nil {
			me.mu.Lock()
			for _, n := range m.R.Nodes {
				me.responseNode(n)
			}
			me.mu.Unlock()

			if vs := m.R.Values; len(vs) != 0 {
				nodeInfo := NodeInfo{
					Addr: t.remoteAddr,
				}
				copy(nodeInfo.ID[:], m.SenderID())
				select {
				case me.values <- PeersValues{
					Peers: func() (ret []Peer) {
						for _, cp := range vs {
							ret = append(ret, Peer(cp))
						}
						return
					}(),
					NodeInfo: nodeInfo,
				}:
				case <-me.stop:
				}
			}

			me.maybeAnnouncePeer(addr, m.R.Token, m.SenderID())
		}

		me.mu.Lock()
		me.transactionClosed()
		me.mu.Unlock()
	})
	return nil
}

// Corresponds to the "values" key in a get_peers KRPC response. A list of
// peers that a node has reported as being in the swarm for a queried info
// hash.
type PeersValues struct {
	Peers    []Peer // Peers given in get_peers response.
	NodeInfo        // The node that gave the response.
}

// Stop the announce.
func (me *Announce) Close() {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.close()
}

func (ps *Announce) close() {
	select {
	case <-ps.stop:
	default:
		close(ps.stop)
	}
}
<commit_msg>Fix #47<commit_after>package dht

// get_peers and announce_peers.

import (
	"log"
	"time"

	"github.com/anacrolix/missinggo"
	"github.com/anacrolix/sync"
	"github.com/willf/bloom"

	"github.com/anacrolix/torrent/logonce"
)

// Maintains state for an ongoing Announce operation. An Announce is started
// by calling Server.Announce.
type Announce struct {
	mu    sync.Mutex
	Peers chan PeersValues
	// Inner chan is set to nil when on close.
	values              chan PeersValues
	stop                chan struct{}
	triedAddrs          *bloom.BloomFilter
	pending             int
	server              *Server
	infoHash            string
	numContacted        int
	announcePort        int
	announcePortImplied bool
}

// Returns the number of distinct remote addresses the announce has queried.
func (me *Announce) NumContacted() int {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me.numContacted
}

// This is kind of the main thing you want to do with DHT. It traverses the
// graph toward nodes that store peers for the infohash, streaming them to the
// caller, and announcing the local node to each node if allowed and
// specified.
func (s *Server) Announce(infoHash string, port int, impliedPort bool) (*Announce, error) {
	s.mu.Lock()
	startAddrs := func() (ret []dHTAddr) {
		for _, n := range s.closestGoodNodes(160, infoHash) {
			ret = append(ret, n.addr)
		}
		return
	}()
	s.mu.Unlock()
	if len(startAddrs) == 0 {
		addrs, err := bootstrapAddrs(s.bootstrapNodes)
		if err != nil {
			return nil, err
		}
		for _, addr := range addrs {
			startAddrs = append(startAddrs, newDHTAddr(addr))
		}
	}
	disc := &Announce{
		Peers:               make(chan PeersValues, 100),
		stop:                make(chan struct{}),
		values:              make(chan PeersValues),
		triedAddrs:          bloom.NewWithEstimates(1000, 0.5),
		server:              s,
		infoHash:            infoHash,
		announcePort:        port,
		announcePortImplied: impliedPort,
	}
	// Function ferries from values to Values until discovery is halted.
	go func() {
		defer close(disc.Peers)
		for {
			select {
			case psv := <-disc.values:
				select {
				case disc.Peers <- psv:
				case <-disc.stop:
					return
				}
			case <-disc.stop:
				return
			}
		}
	}()
	for i, addr := range startAddrs {
		if i != 0 {
			time.Sleep(time.Millisecond)
		}
		disc.mu.Lock()
		disc.contact(addr)
		disc.mu.Unlock()
	}
	return disc, nil
}

func (me *Announce) gotNodeAddr(addr dHTAddr) {
	if missinggo.AddrPort(addr) == 0 {
		// Not a contactable address.
		return
	}
	if me.triedAddrs.Test([]byte(addr.String())) {
		return
	}
	if me.server.ipBlocked(addr.UDPAddr().IP) {
		return
	}
	me.server.mu.Lock()
	if me.server.badNodes.Test([]byte(addr.String())) {
		me.server.mu.Unlock()
		return
	}
	me.server.mu.Unlock()
	me.contact(addr)
}

func (me *Announce) contact(addr dHTAddr) {
	me.numContacted++
	me.triedAddrs.Add([]byte(addr.String()))
	if err := me.getPeers(addr); err != nil {
		log.Printf("error sending get_peers request to %s: %#v", addr, err)
		return
	}
	me.pending++
}

func (me *Announce) transactionClosed() {
	me.pending--
	if me.pending == 0 {
		me.close()
		return
	}
}

func (me *Announce) responseNode(node NodeInfo) {
	me.gotNodeAddr(node.Addr)
}

func (me *Announce) closingCh() chan struct{} {
	return me.stop
}

// Announce to a peer, if appropriate.
func (me *Announce) maybeAnnouncePeer(to dHTAddr, token, peerId string) {
	me.server.mu.Lock()
	defer me.server.mu.Unlock()
	if !me.server.config.NoSecurity {
		if len(peerId) != 20 {
			return
		}
		if !NodeIdSecure(peerId, to.IP()) {
			return
		}
	}
	err := me.server.announcePeer(to, me.infoHash, me.announcePort, token, me.announcePortImplied)
	if err != nil {
		logonce.Stderr.Printf("error announcing peer: %s", err)
	}
}

func (me *Announce) getPeers(addr dHTAddr) error {
	me.server.mu.Lock()
	defer me.server.mu.Unlock()
	t, err := me.server.getPeers(addr, me.infoHash)
	if err != nil {
		return err
	}
	t.SetResponseHandler(func(m Msg, ok bool) {
		// Register suggested nodes closer to the target info-hash.
		if m.R != nil {
			me.mu.Lock()
			for _, n := range m.R.Nodes {
				me.responseNode(n)
			}
			me.mu.Unlock()

			if vs := m.R.Values; len(vs) != 0 {
				nodeInfo := NodeInfo{
					Addr: t.remoteAddr,
				}
				copy(nodeInfo.ID[:], m.SenderID())
				select {
				case me.values <- PeersValues{
					Peers: func() (ret []Peer) {
						for _, cp := range vs {
							ret = append(ret, Peer(cp))
						}
						return
					}(),
					NodeInfo: nodeInfo,
				}:
				case <-me.stop:
				}
			}

			me.maybeAnnouncePeer(addr, m.R.Token, m.SenderID())
		}

		me.mu.Lock()
		me.transactionClosed()
		me.mu.Unlock()
	})
	return nil
}

// Corresponds to the "values" key in a get_peers KRPC response. A list of
// peers that a node has reported as being in the swarm for a queried info
// hash.
type PeersValues struct {
	Peers    []Peer // Peers given in get_peers response.
	NodeInfo        // The node that gave the response.
}

// Stop the announce.
func (me *Announce) Close() {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.close()
}

func (ps *Announce) close() {
	select {
	case <-ps.stop:
	default:
		close(ps.stop)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package api

import (
	"fmt"
	"net/http"
	"strconv"
	"time"
	"sync"
	"sync/atomic"

	"github.com/pagodabox/nanobox-boxfile"
	"github.com/pagodabox/nanobox-server/config"
	"github.com/pagodabox/nanobox-server/util"
)

var execWait = sync.WaitGroup{}
var waiting = int64(0)

var execKeys = map[string]string{}

func (api *API) Suspend(rw http.ResponseWriter, req *http.Request) {
	if waiting == 0 {
		return
	}

	writeBody(map[string]string{"error": fmt.Sprintf("still have %d connected consoles", waiting)}, rw, http.StatusNotAcceptable)
}

func (api *API) Run(rw http.ResponseWriter, req *http.Request) {
	name := req.FormValue("container")
	if name != "" {
		api.Exec(rw, req)
		return
	}

	box := mergedBox()

	containerControl := false
	// if there is no exec 1 it needs to be created and this thread needs to remember
	// to shut it down when its done conatinerControl is used for that purpose
	container, err := util.GetContainer("exec1")
	if err != nil {
		containerControl = true
		cmd := []string{"/bin/sleep", "365d"}

		image := "nanobox/build"
		if stab := box.Node("build").StringValue("stability"); stab != "" {
			image = image + ":" + stab
		}

		container, err = util.CreateContainer(util.CreateConfig{Image: image, Category: "exec", Name: "exec1", Cmd: cmd})
		if err != nil {
			rw.Write([]byte(err.Error()))
			return
		}
	}

	api.Exec(rw, req)

	if containerControl {
		execWait.Wait()
		util.RemoveContainer(container.ID)
	}
}

func (api *API) LibDirs(rw http.ResponseWriter, req *http.Request) {
	writeBody(util.LibDirs(), rw, http.StatusOK)
}

func (api *API) FileChange(rw http.ResponseWriter, req *http.Request) {
	util.Touch(req.FormValue("filename"))
	writeBody(nil, rw, http.StatusOK)
}

func (api *API) KillRun(rw http.ResponseWriter, req *http.Request) {
	fmt.Printf("signal recieved: %s\n", req.FormValue("signal"))
	err := util.KillContainer("exec1", req.FormValue("signal"))
	fmt.Println(err)
}

func (api *API) ResizeRun(rw http.ResponseWriter, req *http.Request) {
	if req.FormValue("container") != "" {
		api.ResizeExec(rw, req)
		return
	}
	h, _ := strconv.Atoi(req.FormValue("h"))
	w, _ := strconv.Atoi(req.FormValue("w"))
	if h == 0 || w == 0 {
		return
	}
	err := util.ResizeContainerTTY("exec1", h, w)
	fmt.Println(err)
}

// proxy an exec request to docker. This allows us to have the same
// exec power but with added security.
func (api *API) Exec(rw http.ResponseWriter, req *http.Request) {
	execWait.Add(1)
	atomic.AddInt64(&waiting, 1)
	defer execWait.Done()
	defer atomic.AddInt64(&waiting, -1)
	name := req.FormValue("container")
	if name == "" {
		name = "exec1"
	}

	conn, _, err := rw.(http.Hijacker).Hijack()
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write([]byte(err.Error()))
		return
	}
	defer conn.Close()

	cmd := []string{"/bin/bash"}
	if additionalCmd := req.FormValue("cmd"); additionalCmd != "" {
		cmd = append(cmd, "-c", additionalCmd)
	}

	container, err := util.GetContainer(name)
	if err != nil {
		conn.Write([]byte(err.Error()))
		return
	}

	// Flush the options to make sure the client sets the raw mode
	conn.Write([]byte{})

	exec, err := util.CreateExec(container.ID, cmd, true, true, true)
	if err == nil {
		execKeys[name] = exec.ID
		defer delete(execKeys, name)
		util.RunExec(exec, conn, conn, conn)
	}
}

// necessary for anything using a windowing system through the exec.
func (api *API) ResizeExec(rw http.ResponseWriter, req *http.Request) {
	name := req.FormValue("container")
	if execKeys[name] == "" {
		time.Sleep(1 * time.Second)
	}
	if name == "" || execKeys[name] == "" {
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	h, _ := strconv.Atoi(req.FormValue("h"))
	w, _ := strconv.Atoi(req.FormValue("w"))
	if h == 0 || w == 0 {
		return
	}

	err := util.ResizeExecTTY(execKeys[name], h, w)
	fmt.Println(err)
}

func mergedBox() (box boxfile.Boxfile) {
	box = boxfile.NewFromPath("/vagrant/code/" + config.App + "/Boxfile")
	if out, err := util.ExecHook("boxfile", "build1", map[string]interface{}{}); err == nil {
		box.Merge(boxfile.New([]byte(out)))
	}
	return
}
<commit_msg>update<commit_after>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package api

import (
	"fmt"
	"net/http"
	"strconv"
	"time"
	"sync"
	"sync/atomic"

	"github.com/pagodabox/nanobox-boxfile"
	"github.com/pagodabox/nanobox-server/config"
	"github.com/pagodabox/nanobox-server/util"
)

var execWait = sync.WaitGroup{}
var waiting = int64(0)

var execKeys = map[string]string{}

func (api *API) Suspend(rw http.ResponseWriter, req *http.Request) {
	if waiting == 0 {
		return
	}

	writeBody(map[string]string{"error": fmt.Sprintf("Still have %d connected consoles", waiting)}, rw, http.StatusNotAcceptable)
}

func (api *API) Run(rw http.ResponseWriter, req *http.Request) {
	name := req.FormValue("container")
	if name != "" {
		api.Exec(rw, req)
		return
	}

	box := mergedBox()

	containerControl := false
	// if there is no exec 1 it needs to be created and this thread needs to remember
	// to shut it down when its done conatinerControl is used for that purpose
	container, err := util.GetContainer("exec1")
	if err != nil {
		containerControl = true
		cmd := []string{"/bin/sleep", "365d"}

		image := "nanobox/build"
		if stab := box.Node("build").StringValue("stability"); stab != "" {
			image = image + ":" + stab
		}

		container, err = util.CreateContainer(util.CreateConfig{Image: image, Category: "exec", Name: "exec1", Cmd: cmd})
		if err != nil {
			rw.Write([]byte(err.Error()))
			return
		}
	}

	api.Exec(rw, req)

	if containerControl {
		execWait.Wait()
		util.RemoveContainer(container.ID)
	}
}

func (api *API) LibDirs(rw http.ResponseWriter, req *http.Request) {
	writeBody(util.LibDirs(), rw, http.StatusOK)
}

func (api *API) FileChange(rw http.ResponseWriter, req *http.Request) {
	util.Touch(req.FormValue("filename"))
	writeBody(nil, rw, http.StatusOK)
}

func (api *API) KillRun(rw http.ResponseWriter, req *http.Request) {
	fmt.Printf("signal recieved: %s\n", req.FormValue("signal"))
	err := util.KillContainer("exec1", req.FormValue("signal"))
	fmt.Println(err)
}

func (api *API) ResizeRun(rw http.ResponseWriter, req *http.Request) {
	if req.FormValue("container") != "" {
		api.ResizeExec(rw, req)
		return
	}
	h, _ := strconv.Atoi(req.FormValue("h"))
	w, _ := strconv.Atoi(req.FormValue("w"))
	if h == 0 || w == 0 {
		return
	}
	err := util.ResizeContainerTTY("exec1", h, w)
	fmt.Println(err)
}

// proxy an exec request to docker. This allows us to have the same
// exec power but with added security.
func (api *API) Exec(rw http.ResponseWriter, req *http.Request) {
	execWait.Add(1)
	atomic.AddInt64(&waiting, 1)
	defer execWait.Done()
	defer atomic.AddInt64(&waiting, -1)
	name := req.FormValue("container")
	if name == "" {
		name = "exec1"
	}

	conn, _, err := rw.(http.Hijacker).Hijack()
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write([]byte(err.Error()))
		return
	}
	defer conn.Close()

	cmd := []string{"/bin/bash"}
	if additionalCmd := req.FormValue("cmd"); additionalCmd != "" {
		cmd = append(cmd, "-c", additionalCmd)
	}

	container, err := util.GetContainer(name)
	if err != nil {
		conn.Write([]byte(err.Error()))
		return
	}

	// Flush the options to make sure the client sets the raw mode
	conn.Write([]byte{})

	exec, err := util.CreateExec(container.ID, cmd, true, true, true)
	if err == nil {
		execKeys[name] = exec.ID
		defer delete(execKeys, name)
		util.RunExec(exec, conn, conn, conn)
	}
}

// necessary for anything using a windowing system through the exec.
func (api *API) ResizeExec(rw http.ResponseWriter, req *http.Request) {
	name := req.FormValue("container")
	if execKeys[name] == "" {
		time.Sleep(1 * time.Second)
	}
	if name == "" || execKeys[name] == "" {
		rw.WriteHeader(http.StatusNotFound)
		return
	}

	h, _ := strconv.Atoi(req.FormValue("h"))
	w, _ := strconv.Atoi(req.FormValue("w"))
	if h == 0 || w == 0 {
		return
	}

	err := util.ResizeExecTTY(execKeys[name], h, w)
	fmt.Println(err)
}

func mergedBox() (box boxfile.Boxfile) {
	box = boxfile.NewFromPath("/vagrant/code/" + config.App + "/Boxfile")
	if out, err := util.ExecHook("boxfile", "build1", map[string]interface{}{}); err == nil {
		box.Merge(boxfile.New([]byte(out)))
	}
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package api

import (
	"bytes"
	l4g "code.google.com/p/log4go"
	"fmt"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/s3"
	"github.com/gorilla/mux"
	"github.com/mattermost/platform/model"
	"github.com/mattermost/platform/utils"
	"github.com/nfnt/resize"
	"image"
	_ "image/gif"
	_ "golang.org/x/image/bmp"
	"image/jpeg"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func InitFile(r *mux.Router) {
	l4g.Debug("Initializing post api routes")

	sr := r.PathPrefix("/files").Subrouter()
	sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
	sr.Handle("/get/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+\\.[A-Za-z0-9]{3,}}", ApiAppHandler(getFile)).Methods("GET")
	sr.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("POST")
}

func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
	if !utils.IsS3Configured() {
		c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "")
		c.Err.StatusCode = http.StatusNotImplemented
		return
	}

	err := r.ParseMultipartForm(model.MAX_FILE_SIZE)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var auth aws.Auth
	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

	m := r.MultipartForm

	props := m.Value

	if len(props["channel_id"]) == 0 {
		c.SetInvalidParam("uploadFile", "channel_id")
		return
	}
	channelId := props["channel_id"][0]
	if len(channelId) == 0 {
		c.SetInvalidParam("uploadFile", "channel_id")
		return
	}

	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)

	files := m.File["files"]

	resStruct := &model.FileUploadResponse{
		Filenames: []string{}}

	imageNameList := []string{}
	imageDataList := [][]byte{}

	if !c.HasPermissionsToChannel(cchan, "uploadFile") {
		return
	}

	for i, _ := range files {
		file, err := files[i].Open()
		defer file.Close()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		buf := bytes.NewBuffer(nil)
		io.Copy(buf, file)

		ext := filepath.Ext(files[i].Filename)

		uid := model.NewId()

		path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename

		if model.IsFileExtImage(ext) {
			options := s3.Options{}
			err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options)
			imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
			imageDataList = append(imageDataList, buf.Bytes())
		} else {
			options := s3.Options{}
			err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options)
		}

		if err != nil {
			c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error())
			return
		}

		fileUrl := c.TeamUrl + "/api/v1/files/get/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename)
		resStruct.Filenames = append(resStruct.Filenames, fileUrl)
	}

	fireAndForgetHandleImages(imageNameList, imageDataList, c.Session.TeamId, channelId, c.Session.UserId)

	w.Write([]byte(resStruct.ToJson()))
}

func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {

	go func() {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

		dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"

		for i, filename := range filenames {
			name := filename[:strings.LastIndex(filename, ".")]
			go func() {
				// Decode image bytes into Image object
				img, _, err := image.Decode(bytes.NewReader(fileData[i]))
				if err != nil {
					l4g.Error("Unable to decode image channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
					return
				}

				// Decode image config
				imgConfig, _, err := image.DecodeConfig(bytes.NewReader(fileData[i]))
				if err != nil {
					l4g.Error("Unable to decode image config channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
					return
				}

				// Create thumbnail
				go func() {
					var thumbnail image.Image
					if imgConfig.Width > int(utils.Cfg.ImageSettings.ThumbnailWidth) {
						thumbnail = resize.Resize(utils.Cfg.ImageSettings.ThumbnailWidth, utils.Cfg.ImageSettings.ThumbnailHeight, img, resize.NearestNeighbor)
					} else {
						thumbnail = img
					}

					buf := new(bytes.Buffer)
					err = jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90})
					if err != nil {
						l4g.Error("Unable to encode image as jpeg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}

					// Upload thumbnail to S3
					options := s3.Options{}
					err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
					if err != nil {
						l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}
				}()

				// Create preview
				go func() {
					var preview image.Image
					if imgConfig.Width > int(utils.Cfg.ImageSettings.PreviewWidth) {
						preview = resize.Resize(utils.Cfg.ImageSettings.PreviewWidth, utils.Cfg.ImageSettings.PreviewHeight, img, resize.NearestNeighbor)
					} else {
						preview = img
					}

					buf := new(bytes.Buffer)
					err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})

					//err = png.Encode(buf, preview)
					if err != nil {
						l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}

					// Upload preview to S3
					options := s3.Options{}
					err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
					if err != nil {
						l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}
				}()
			}()
		}
	}()
}

type ImageGetResult struct {
	Error     error
	ImageData []byte
}

func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
	if !utils.IsS3Configured() {
		c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "")
		c.Err.StatusCode = http.StatusNotImplemented
		return
	}

	params := mux.Vars(r)

	channelId := params["channel_id"]
	if len(channelId) != 26 {
		c.SetInvalidParam("getFile", "channel_id")
		return
	}

	userId := params["user_id"]
	if len(userId) != 26 {
		c.SetInvalidParam("getFile", "user_id")
		return
	}

	filename := params["filename"]
	if len(filename) == 0 {
		c.SetInvalidParam("getFile", "filename")
		return
	}

	hash := r.URL.Query().Get("h")
	data := r.URL.Query().Get("d")
	teamId := r.URL.Query().Get("t")

	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)

	var auth aws.Auth
	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

	path := ""
	if len(teamId) == 26 {
		path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
	} else {
		path = "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
	}

	fileData := make(chan []byte)
	asyncGetFile(bucket, path, fileData)

	if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 {
		if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) {
			c.Err = model.NewAppError("getFile", "The public link does not appear to be valid", "")
			return
		}
		props := model.MapFromJson(strings.NewReader(data))

		t, err := strconv.ParseInt(props["time"], 10, 64)
		if err != nil || model.GetMillis()-t > 1000*60*60*24*7 { // one week
			c.Err = model.NewAppError("getFile", "The public link has expired", "")
			return
		}
	} else if !c.HasPermissionsToChannel(cchan, "getFile") {
		return
	}

	f := <-fileData

	if f == nil {
		var f2 []byte
		tries := 0
		for {
			time.Sleep(3000 * time.Millisecond)
			tries++

			asyncGetFile(bucket, path, fileData)
			f2 = <-fileData

			if f2 != nil {
				w.Header().Set("Cache-Control", "max-age=2592000, public")
				w.Header().Set("Content-Length", strconv.Itoa(len(f2)))
				w.Write(f2)
				return
			} else if tries >= 2 {
				break
			}
		}

		c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path)
		c.Err.StatusCode = http.StatusNotFound
		return
	}

	w.Header().Set("Cache-Control", "max-age=2592000, public")
	w.Header().Set("Content-Length", strconv.Itoa(len(f)))
	w.Write(f)
}

func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) {
	go func() {
		data, getErr := bucket.Get(path)
		if getErr != nil {
			fileData <- nil
		} else {
			fileData <- data
		}
	}()
}

func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
	if !utils.Cfg.TeamSettings.AllowPublicLink {
		c.Err = model.NewAppError("getPublicLink", "Public links have been disabled", "")
		c.Err.StatusCode = http.StatusForbidden
	}

	if !utils.IsS3Configured() {
		c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "")
		c.Err.StatusCode = http.StatusNotImplemented
		return
	}

	props := model.MapFromJson(r.Body)

	filename := props["filename"]
	if len(filename) == 0 {
		c.SetInvalidParam("getPublicLink", "filename")
		return
	}

	matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1)
	if len(matches) == 0 || len(matches[0]) < 5 {
		c.SetInvalidParam("getPublicLink", "filename")
		return
	}

	getType := matches[0][1]
	channelId := matches[0][2]
	userId := matches[0][3]
	filename = matches[0][4]

	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)

	newProps := make(map[string]string)
	newProps["filename"] = filename
	newProps["time"] = fmt.Sprintf("%v", model.GetMillis())

	data := model.MapToJson(newProps)
	hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt))

	url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.TeamUrl, getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId)

	if !c.HasPermissionsToChannel(cchan, "getPublicLink") {
		return
	}

	rData := make(map[string]string)
	rData["public_link"] = url

	w.Write([]byte(model.MapToJson(rData)))
}
<commit_msg>Changed image resizing for both the preview and thumbnail to use Lanczos interpolation instead of simple nearest neighbour<commit_after>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package api

import (
	"bytes"
	l4g "code.google.com/p/log4go"
	"fmt"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/s3"
	"github.com/gorilla/mux"
	"github.com/mattermost/platform/model"
	"github.com/mattermost/platform/utils"
	"github.com/nfnt/resize"
	_ "golang.org/x/image/bmp"
	"image"
	_ "image/gif"
	"image/jpeg"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func InitFile(r *mux.Router) {
	l4g.Debug("Initializing post api routes")

	sr := r.PathPrefix("/files").Subrouter()
	sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
	sr.Handle("/get/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+\\.[A-Za-z0-9]{3,}}", ApiAppHandler(getFile)).Methods("GET")
	sr.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("POST")
}

func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
	if !utils.IsS3Configured() {
		c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "")
		c.Err.StatusCode = http.StatusNotImplemented
		return
	}

	err := r.ParseMultipartForm(model.MAX_FILE_SIZE)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var auth aws.Auth
	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

	m := r.MultipartForm

	props := m.Value

	if len(props["channel_id"]) == 0 {
		c.SetInvalidParam("uploadFile", "channel_id")
		return
	}
	channelId := props["channel_id"][0]
	if len(channelId) == 0 {
		c.SetInvalidParam("uploadFile", "channel_id")
		return
	}

	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)

	files := m.File["files"]

	resStruct := &model.FileUploadResponse{
		Filenames: []string{}}

	imageNameList := []string{}
	imageDataList := [][]byte{}

	if !c.HasPermissionsToChannel(cchan, "uploadFile") {
		return
	}

	for i, _ := range files {
		file, err := files[i].Open()
		defer file.Close()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		buf := bytes.NewBuffer(nil)
		io.Copy(buf, file)

		ext := filepath.Ext(files[i].Filename)

		uid := model.NewId()

		path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename

		if model.IsFileExtImage(ext) {
			options := s3.Options{}
			err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options)
			imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
			imageDataList = append(imageDataList, buf.Bytes())
		} else {
			options := s3.Options{}
			err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options)
		}

		if err != nil {
			c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error())
			return
		}

		fileUrl := c.TeamUrl + "/api/v1/files/get/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename)
		resStruct.Filenames = append(resStruct.Filenames, fileUrl)
	}

	fireAndForgetHandleImages(imageNameList, imageDataList, c.Session.TeamId, channelId, c.Session.UserId)

	w.Write([]byte(resStruct.ToJson()))
}

func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {

	go func() {
		var auth aws.Auth
		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

		dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"

		for i, filename := range filenames {
			name := filename[:strings.LastIndex(filename, ".")]
			go func() {
				// Decode image bytes into Image object
				img, _, err := image.Decode(bytes.NewReader(fileData[i]))
				if err != nil {
					l4g.Error("Unable to decode image channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
					return
				}

				// Decode image config
				imgConfig, _, err := image.DecodeConfig(bytes.NewReader(fileData[i]))
				if err != nil {
					l4g.Error("Unable to decode image config channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
					return
				}

				// Create thumbnail
				go func() {
					var thumbnail image.Image
					if imgConfig.Width > int(utils.Cfg.ImageSettings.ThumbnailWidth) {
						thumbnail = resize.Resize(utils.Cfg.ImageSettings.ThumbnailWidth, utils.Cfg.ImageSettings.ThumbnailHeight, img, resize.Lanczos3)
					} else {
						thumbnail = img
					}

					buf := new(bytes.Buffer)
					err = jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90})
					if err != nil {
						l4g.Error("Unable to encode image as jpeg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}

					// Upload thumbnail to S3
					options := s3.Options{}
					err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
					if err != nil {
						l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}
				}()

				// Create preview
				go func() {
					var preview image.Image
					if imgConfig.Width > int(utils.Cfg.ImageSettings.PreviewWidth) {
						preview = resize.Resize(utils.Cfg.ImageSettings.PreviewWidth, utils.Cfg.ImageSettings.PreviewHeight, img, resize.Lanczos3)
					} else {
						preview = img
					}

					buf := new(bytes.Buffer)
					err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})

					//err = png.Encode(buf, preview)
					if err != nil {
						l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}

					// Upload preview to S3
					options := s3.Options{}
					err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
					if err != nil {
						l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
						return
					}
				}()
			}()
		}
	}()
}

type ImageGetResult struct {
	Error     error
	ImageData []byte
}

func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
	if !utils.IsS3Configured() {
		c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "")
		c.Err.StatusCode = http.StatusNotImplemented
		return
	}

	params := mux.Vars(r)

	channelId := params["channel_id"]
	if len(channelId) != 26 {
		c.SetInvalidParam("getFile", "channel_id")
		return
	}

	userId := params["user_id"]
	if len(userId) != 26 {
		c.SetInvalidParam("getFile", "user_id")
		return
	}

	filename := params["filename"]
	if len(filename) == 0 {
		c.SetInvalidParam("getFile", "filename")
		return
	}

	hash := r.URL.Query().Get("h")
	data := r.URL.Query().Get("d")
	teamId := r.URL.Query().Get("t")

	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)

	var auth aws.Auth
	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey

	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)

	path := ""
	if len(teamId) == 26 {
		path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
	} else {
		path = "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
	}

	fileData := make(chan []byte)
	asyncGetFile(bucket, path, fileData)

	if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 {
		if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) {
			c.Err = model.NewAppError("getFile", "The public link does not appear to be valid", "")
			return
		}
		props := model.MapFromJson(strings.NewReader(data))

		t, err := strconv.ParseInt(props["time"], 10, 64)
		if err != nil || model.GetMillis()-t > 1000*60*60*24*7 { // one week
			c.Err = model.NewAppError("getFile", "The public link has expired", "")
			return
		}
	} else if !c.HasPermissionsToChannel(cchan, "getFile") {
		return
	}

	f := <-fileData

	if f == nil {
		var f2 []byte
		tries := 0
		for {
			time.Sleep(3000 * time.Millisecond)
			tries++

			asyncGetFile(bucket, path, fileData)
			f2 = <-fileData

			if f2 != nil {
				w.Header().Set("Cache-Control", "max-age=2592000, public")
				w.Header().Set("Content-Length", strconv.Itoa(len(f2)))
				w.Write(f2)
				return
			} else if tries >= 2 {
				break
			}
		}

		c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path)
		c.Err.StatusCode = http.StatusNotFound
		return
	}

	w.Header().Set("Cache-Control", "max-age=2592000, public")
	w.Header().Set("Content-Length", strconv.Itoa(len(f)))
	w.Write(f)
}

func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) {
	go func() {
		data, getErr := bucket.Get(path)
		if getErr != nil {
			fileData <- nil
		} else {
			fileData <- data
		}
	}()
}

func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
	if !utils.Cfg.TeamSettings.AllowPublicLink {
		c.Err = model.NewAppError("getPublicLink", "Public links have been disabled", "")
		c.Err.StatusCode = http.StatusForbidden
	}

	if !utils.IsS3Configured() {
		c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "")
		c.Err.StatusCode = http.StatusNotImplemented
		return
	}

	props := model.MapFromJson(r.Body)

	filename := props["filename"]
	if len(filename) == 0 {
		c.SetInvalidParam("getPublicLink", "filename")
		return
	}

	matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1)
	if len(matches) == 0 || len(matches[0]) < 5 {
		c.SetInvalidParam("getPublicLink", "filename")
		return
	}

	getType := matches[0][1]
	channelId := matches[0][2]
	userId := matches[0][3]
	filename = matches[0][4]

	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)

	newProps := make(map[string]string)
	newProps["filename"] = filename
	newProps["time"] = fmt.Sprintf("%v", model.GetMillis())

	data := model.MapToJson(newProps)
	hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt))

	url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.TeamUrl, getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId)

	if !c.HasPermissionsToChannel(cchan, "getPublicLink") {
		return
	}

	rData := make(map[string]string)
	rData["public_link"] = url

	w.Write([]byte(model.MapToJson(rData)))
}
<|endoftext|>