<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package fs contains an HTTP file system that works with zip contents.
package fs

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/magic"
	"github.com/hashicorp/go-multierror"
)

var assetsClient = &http.Client{
	Timeout: 30 * time.Second,
}

var globalAssets sync.Map // {context:path -> *Asset}

const sumLen = 10
const defaultContext = "default"

type AssetOption struct {
	Name     string `json:"name"`
	Context  string `json:"context"`
	URL      string `json:"url"`
	Shasum   string `json:"shasum"`
	IsCustom bool   `json:"is_custom,omitempty"`
}

// Asset holds unzipped read-only file contents and file metadata.
type Asset struct {
	AssetOption
	Etag        string `json:"etag"`
	NameWithSum string `json:"name_with_sum"`
	Mime        string `json:"mime"`

	zippedData   []byte
	zippedSize   string
	unzippedData []byte
	unzippedSize string
}

func (f *Asset) Size() string {
	return f.unzippedSize
}
func (f *Asset) Reader() *bytes.Reader {
	return bytes.NewReader(f.unzippedData)
}

func (f *Asset) GzipSize() string {
	return f.zippedSize
}
func (f *Asset) GzipReader() *bytes.Reader {
	return bytes.NewReader(f.zippedData)
}

// Register registers zip contents data, later used to
// initialize the statik file system.
func Register(zipData string) {
	if zipData == "" {
		panic("statik/fs: no zip data registered")
	}
	if err := unzip([]byte(zipData)); err != nil {
		panic(fmt.Errorf("statik/fs: error unzipping data: %s", err))
	}
}

type Cache interface {
	Get(key string) (io.Reader, bool)
	Set(key string, data []byte, expiration time.Duration) bool
}

func RegisterCustomExternals(cache Cache, opts []AssetOption, maxTryCount int) error {
	if len(opts) == 0 {
		return nil
	}

	assetsCh := make(chan AssetOption)
	doneCh := make(chan error)

	for i := 0; i < len(opts); i++ {
		go func() {
			var err error
			sleepDuration := 500 * time.Millisecond
			opt := <-assetsCh

			for tryCount := 0; tryCount < maxTryCount+1; tryCount++ {
				err = registerCustomExternal(cache, opt)
				if err == nil {
					break
				}
				logger.WithNamespace("statik").
					Errorf("Could not load asset from %q, retrying in %s", opt.URL, sleepDuration)
				time.Sleep(sleepDuration)
				sleepDuration *= 2
			}

			doneCh <- err
		}()
	}

	for _, opt := range opts {
		assetsCh <- opt
	}
	close(assetsCh)

	var errm error
	for i := 0; i < len(opts); i++ {
		if err := <-doneCh; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	return errm
}

func registerCustomExternal(cache Cache, opt AssetOption) error {
	if opt.Context == "" {
		logger.WithNamespace("custom assets").Warningf("Could not load asset %s with empty context\n", opt.URL)
		return nil
	}

	name := normalizeAssetName(opt.Name)
	if currentAsset, ok := Get(name, opt.Context); ok {
		if currentAsset.Shasum == opt.Shasum {
			return nil
		}
	}

	opt.IsCustom = true

	assetURL := opt.URL
	key := fmt.Sprintf("assets:%s:%s:%s", opt.Context, name, opt.Shasum)

	var body io.Reader
	var ok, storeInCache bool
	if body, ok = cache.Get(key); !ok {
		u, err := url.Parse(assetURL)
		if err != nil {
			return err
		}

		switch u.Scheme {
		case "http", "https":
			req, err := http.NewRequest(http.MethodGet, assetURL, nil)
			if err != nil {
				return err
			}
			res, err := assetsClient.Do(req)
			if err != nil {
				return err
			}
			defer res.Body.Close()
			if res.StatusCode != http.StatusOK {
				return fmt.Errorf("could not load external asset on %s: status code %d", assetURL, res.StatusCode)
			}
			body = res.Body
		case "file":
			f, err := os.Open(u.Path)
			if err != nil {
				return err
			}
			defer f.Close()
			body = f
		default:
			return fmt.Errorf("does not support externals assets with scheme %q", u.Scheme)
		}

		storeInCache = true
	}

	h := sha256.New()
	zippedDataBuf := new(bytes.Buffer)
	gw := gzip.NewWriter(zippedDataBuf)

	teeReader := io.TeeReader(body, io.MultiWriter(h, gw))
	unzippedData, err := ioutil.ReadAll(teeReader)
	if err != nil {
		return err
	}
	if errc := gw.Close(); errc != nil {
		return errc
	}

	sum := h.Sum(nil)

	if opt.Shasum == "" {
		opt.Shasum = hex.EncodeToString(sum)
		log := logger.WithNamespace("custom_external")
		log.Warnf("shasum was not provided for file %s, inserting unsafe content %s: %s",
			opt.Name, opt.URL, opt.Shasum)
	}

	if hex.EncodeToString(sum) != opt.Shasum {
		return fmt.Errorf("external content checksum do not match: expected %s got %x on url %s",
			opt.Shasum, sum, assetURL)
	}

	if storeInCache {
		expiration := 30 * 24 * time.Hour
		cache.Set(key, unzippedData, expiration)
	}

	asset := newAsset(opt, zippedDataBuf.Bytes(), unzippedData)
	storeAsset(asset)
	return nil
}

func unzip(data []byte) (err error) {
	for {
		block, rest := pem.Decode(data)
		if block == nil {
			break
		}
		var zippedData, unzippedData []byte
		zippedData = block.Bytes
		var gr *gzip.Reader
		gr, err = gzip.NewReader(bytes.NewReader(block.Bytes))
		if err != nil {
			return
		}
		h := sha256.New()
		r := io.TeeReader(gr, h)
		unzippedData, err = ioutil.ReadAll(r)
		if err != nil {
			return
		}
		if err = gr.Close(); err != nil {
			return
		}

		name := block.Headers["Name"]
		opt := AssetOption{
			Name:    name,
			Context: defaultContext,
			Shasum:  hex.EncodeToString(h.Sum(nil)),
		}
		asset := newAsset(opt, zippedData, unzippedData)
		storeAsset(asset)
		data = rest
	}
	return
}

func normalizeAssetName(name string) string {
	return path.Join("/", name)
}

func newAsset(opt AssetOption, zippedData, unzippedData []byte) *Asset {
	mime := magic.MIMETypeByExtension(path.Ext(opt.Name))
	if mime == "" {
		mime = magic.MIMEType(unzippedData)
	}
	if mime == "" {
		mime = "application/octet-stream"
	}

	sumx := opt.Shasum
	etag := fmt.Sprintf(`"%s"`, sumx[:sumLen])

	opt.Name = normalizeAssetName(opt.Name)

	nameWithSum := opt.Name
	nameBase := path.Base(opt.Name)
	if off := strings.IndexByte(nameBase, '.'); off >= 0 {
		nameDir := path.Dir(opt.Name)
		nameWithSum = path.Join("/", nameDir, nameBase[:off]+"."+sumx[:sumLen]+nameBase[off:])
	}

	return &Asset{
		AssetOption: opt,
		Etag:        etag,
		NameWithSum: nameWithSum,
		Mime:        mime,
		zippedData:  zippedData,
		zippedSize:  strconv.Itoa(len(zippedData)),

		unzippedData: unzippedData,
		unzippedSize: strconv.Itoa(len(unzippedData)),
	}
}

// threadsafe
func storeAsset(asset *Asset) {
	context := asset.Context
	if context == "" {
		context = defaultContext
	}
	contextKey := marshalContextKey(context, asset.Name)
	globalAssets.Store(contextKey, asset)
}

func Get(name string, context ...string) (*Asset, bool) {
	var ctx string
	if len(context) > 0 && context[0] != "" {
		ctx = context[0]
	} else {
		ctx = defaultContext
	}
	asset, ok := globalAssets.Load(marshalContextKey(ctx, name))
	if !ok {
		return nil, false
	}
	return asset.(*Asset), true
}

func Open(name string, context ...string) (*bytes.Reader, error) {
	f, ok := Get(name, context...)
	if ok {
		return f.Reader(), nil
	}
	return nil, os.ErrNotExist
}

func Foreach(predicate func(name, context string, f *Asset)) {
	globalAssets.Range(func(contextKey interface{}, v interface{}) bool {
		context, name, _ := unMarshalContextKey(contextKey.(string))
		predicate(name, context, v.(*Asset))
		return true
	})
}

func marshalContextKey(context, name string) (marshaledKey string) {
	return context + ":" + name
}

func unMarshalContextKey(contextKey string) (context string, name string, err error) {
	unmarshaled := strings.SplitN(contextKey, ":", 2)
	if len(unmarshaled) != 2 {
		panic("statik/fs: the contextKey is malformed")
	}
	return unmarshaled[0], unmarshaled[1], nil
}
<commit_msg>Recompute the cache key when shasum is known<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package fs contains an HTTP file system that works with zip contents.
package fs

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/magic"
	"github.com/hashicorp/go-multierror"
)

var assetsClient = &http.Client{
	Timeout: 30 * time.Second,
}

var globalAssets sync.Map // {context:path -> *Asset}

const sumLen = 10
const defaultContext = "default"

type AssetOption struct {
	Name     string `json:"name"`
	Context  string `json:"context"`
	URL      string `json:"url"`
	Shasum   string `json:"shasum"`
	IsCustom bool   `json:"is_custom,omitempty"`
}

// Asset holds unzipped read-only file contents and file metadata.
type Asset struct {
	AssetOption
	Etag        string `json:"etag"`
	NameWithSum string `json:"name_with_sum"`
	Mime        string `json:"mime"`

	zippedData   []byte
	zippedSize   string
	unzippedData []byte
	unzippedSize string
}

func (f *Asset) Size() string {
	return f.unzippedSize
}
func (f *Asset) Reader() *bytes.Reader {
	return bytes.NewReader(f.unzippedData)
}

func (f *Asset) GzipSize() string {
	return f.zippedSize
}
func (f *Asset) GzipReader() *bytes.Reader {
	return bytes.NewReader(f.zippedData)
}

// Register registers zip contents data, later used to
// initialize the statik file system.
func Register(zipData string) {
	if zipData == "" {
		panic("statik/fs: no zip data registered")
	}
	if err := unzip([]byte(zipData)); err != nil {
		panic(fmt.Errorf("statik/fs: error unzipping data: %s", err))
	}
}

type Cache interface {
	Get(key string) (io.Reader, bool)
	Set(key string, data []byte, expiration time.Duration) bool
}

func RegisterCustomExternals(cache Cache, opts []AssetOption, maxTryCount int) error {
	if len(opts) == 0 {
		return nil
	}

	assetsCh := make(chan AssetOption)
	doneCh := make(chan error)

	for i := 0; i < len(opts); i++ {
		go func() {
			var err error
			sleepDuration := 500 * time.Millisecond
			opt := <-assetsCh

			for tryCount := 0; tryCount < maxTryCount+1; tryCount++ {
				err = registerCustomExternal(cache, opt)
				if err == nil {
					break
				}
				logger.WithNamespace("statik").
					Errorf("Could not load asset from %q, retrying in %s", opt.URL, sleepDuration)
				time.Sleep(sleepDuration)
				sleepDuration *= 2
			}

			doneCh <- err
		}()
	}

	for _, opt := range opts {
		assetsCh <- opt
	}
	close(assetsCh)

	var errm error
	for i := 0; i < len(opts); i++ {
		if err := <-doneCh; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	return errm
}

func registerCustomExternal(cache Cache, opt AssetOption) error {
	if opt.Context == "" {
		logger.WithNamespace("custom assets").Warningf("Could not load asset %s with empty context\n", opt.URL)
		return nil
	}

	name := normalizeAssetName(opt.Name)
	if currentAsset, ok := Get(name, opt.Context); ok {
		if currentAsset.Shasum == opt.Shasum {
			return nil
		}
	}

	opt.IsCustom = true

	assetURL := opt.URL
	key := fmt.Sprintf("assets:%s:%s:%s", opt.Context, name, opt.Shasum)

	var body io.Reader
	var ok, storeInCache bool
	if body, ok = cache.Get(key); !ok {
		u, err := url.Parse(assetURL)
		if err != nil {
			return err
		}

		switch u.Scheme {
		case "http", "https":
			req, err := http.NewRequest(http.MethodGet, assetURL, nil)
			if err != nil {
				return err
			}
			res, err := assetsClient.Do(req)
			if err != nil {
				return err
			}
			defer res.Body.Close()
			if res.StatusCode != http.StatusOK {
				return fmt.Errorf("could not load external asset on %s: status code %d", assetURL, res.StatusCode)
			}
			body = res.Body
		case "file":
			f, err := os.Open(u.Path)
			if err != nil {
				return err
			}
			defer f.Close()
			body = f
		default:
			return fmt.Errorf("does not support externals assets with scheme %q", u.Scheme)
		}

		storeInCache = true
	}

	h := sha256.New()
	zippedDataBuf := new(bytes.Buffer)
	gw := gzip.NewWriter(zippedDataBuf)

	teeReader := io.TeeReader(body, io.MultiWriter(h, gw))
	unzippedData, err := ioutil.ReadAll(teeReader)
	if err != nil {
		return err
	}
	if errc := gw.Close(); errc != nil {
		return errc
	}

	sum := h.Sum(nil)

	if opt.Shasum == "" {
		opt.Shasum = hex.EncodeToString(sum)
		key = fmt.Sprintf("assets:%s:%s:%s", opt.Context, name, opt.Shasum)
		log := logger.WithNamespace("custom_external")
		log.Warnf("shasum was not provided for file %s, inserting unsafe content %s: %s",
			opt.Name, opt.URL, opt.Shasum)
	}

	if hex.EncodeToString(sum) != opt.Shasum {
		return fmt.Errorf("external content checksum do not match: expected %s got %x on url %s",
			opt.Shasum, sum, assetURL)
	}

	if storeInCache {
		expiration := 30 * 24 * time.Hour
		cache.Set(key, unzippedData, expiration)
	}

	asset := newAsset(opt, zippedDataBuf.Bytes(), unzippedData)
	storeAsset(asset)
	return nil
}

func unzip(data []byte) (err error) {
	for {
		block, rest := pem.Decode(data)
		if block == nil {
			break
		}
		var zippedData, unzippedData []byte
		zippedData = block.Bytes
		var gr *gzip.Reader
		gr, err = gzip.NewReader(bytes.NewReader(block.Bytes))
		if err != nil {
			return
		}
		h := sha256.New()
		r := io.TeeReader(gr, h)
		unzippedData, err = ioutil.ReadAll(r)
		if err != nil {
			return
		}
		if err = gr.Close(); err != nil {
			return
		}

		name := block.Headers["Name"]
		opt := AssetOption{
			Name:    name,
			Context: defaultContext,
			Shasum:  hex.EncodeToString(h.Sum(nil)),
		}
		asset := newAsset(opt, zippedData, unzippedData)
		storeAsset(asset)
		data = rest
	}
	return
}

func normalizeAssetName(name string) string {
	return path.Join("/", name)
}

func newAsset(opt AssetOption, zippedData, unzippedData []byte) *Asset {
	mime := magic.MIMETypeByExtension(path.Ext(opt.Name))
	if mime == "" {
		mime = magic.MIMEType(unzippedData)
	}
	if mime == "" {
		mime = "application/octet-stream"
	}

	sumx := opt.Shasum
	etag := fmt.Sprintf(`"%s"`, sumx[:sumLen])

	opt.Name = normalizeAssetName(opt.Name)

	nameWithSum := opt.Name
	nameBase := path.Base(opt.Name)
	if off := strings.IndexByte(nameBase, '.'); off >= 0 {
		nameDir := path.Dir(opt.Name)
		nameWithSum = path.Join("/", nameDir, nameBase[:off]+"."+sumx[:sumLen]+nameBase[off:])
	}

	return &Asset{
		AssetOption: opt,
		Etag:        etag,
		NameWithSum: nameWithSum,
		Mime:        mime,
		zippedData:  zippedData,
		zippedSize:  strconv.Itoa(len(zippedData)),

		unzippedData: unzippedData,
		unzippedSize: strconv.Itoa(len(unzippedData)),
	}
}

// threadsafe
func storeAsset(asset *Asset) {
	context := asset.Context
	if context == "" {
		context = defaultContext
	}
	contextKey := marshalContextKey(context, asset.Name)
	globalAssets.Store(contextKey, asset)
}

func Get(name string, context ...string) (*Asset, bool) {
	var ctx string
	if len(context) > 0 && context[0] != "" {
		ctx = context[0]
	} else {
		ctx = defaultContext
	}
	asset, ok := globalAssets.Load(marshalContextKey(ctx, name))
	if !ok {
		return nil, false
	}
	return asset.(*Asset), true
}

func Open(name string, context ...string) (*bytes.Reader, error) {
	f, ok := Get(name, context...)
	if ok {
		return f.Reader(), nil
	}
	return nil, os.ErrNotExist
}

func Foreach(predicate func(name, context string, f *Asset)) {
	globalAssets.Range(func(contextKey interface{}, v interface{}) bool {
		context, name, _ := unMarshalContextKey(contextKey.(string))
		predicate(name, context, v.(*Asset))
		return true
	})
}

func marshalContextKey(context, name string) (marshaledKey string) {
	return context + ":" + name
}

func unMarshalContextKey(contextKey string) (context string, name string, err error) {
	unmarshaled := strings.SplitN(contextKey, ":", 2)
	if len(unmarshaled) != 2 {
		panic("statik/fs: the contextKey is malformed")
	}
	return unmarshaled[0], unmarshaled[1], nil
}
<|endoftext|>


<commit_before>package util

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/drud/ddev/pkg/output"
	log "github.com/sirupsen/logrus"
)

// DownloadFile retrieves a file.
func DownloadFile(destPath string, url string, progressBar bool) (err error) {
	// Create the file
	out, err := os.Create(destPath)
	if err != nil {
		return err
	}
	defer CheckClose(out)

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer CheckClose(resp.Body)

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("download link %s returned wrong status code: got %v want %v", url, resp.StatusCode, http.StatusOK)
	}
	reader := resp.Body
	if progressBar {

		bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES).Prefix(filepath.Base(destPath))
		bar.Start()

		// create proxy reader
		reader = bar.NewProxyReader(resp.Body)
		// Writer the body to file
		_, err = io.Copy(out, reader)
		bar.Finish()
	} else {
		_, err = io.Copy(out, reader)
	}

	if err != nil {
		return err
	}

	return nil
}

// HTTPOptions defines the URL and other common HTTP options for EnsureHTTPStatus.
type HTTPOptions struct {
	URL            string
	Username       string
	Password       string
	Timeout        time.Duration
	TickerInterval time.Duration
	ExpectedStatus int
	Headers        map[string]string
}

// NewHTTPOptions returns a new HTTPOptions struct with some sane defaults.
func NewHTTPOptions(URL string) *HTTPOptions {
	o := HTTPOptions{
		URL:            URL,
		TickerInterval: 20,
		Timeout:        60,
		ExpectedStatus: http.StatusOK,
		Headers:        make(map[string]string),
	}
	return &o
}

// EnsureHTTPStatus will verify a URL responds with a given response code within the Timeout period (in seconds)
func EnsureHTTPStatus(o *HTTPOptions) error {

	client := &http.Client{
		Timeout: o.Timeout * time.Second,
	}
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return errors.New("received http redirect")
	}

	req, err := http.NewRequest("GET", o.URL, nil)
	if err != nil {
		return err
	}
	if o.Username != "" && o.Password != "" {
		req.SetBasicAuth(o.Username, o.Password)
	}

	if len(o.Headers) > 0 {
		for header, value := range o.Headers {
			if header == "Host" {
				req.Host = value
				continue
			}
			req.Header.Add(header, value)
		}
	}
	// Make the request
	resp, err := client.Do(req)

	if err == nil {
		defer CheckClose(resp.Body)
		if o.ExpectedStatus != 0 && resp.StatusCode == o.ExpectedStatus {
			return nil
		}
		// Log expected vs. actual if we do not get a match.
		output.UserOut.WithFields(log.Fields{
			"URL":      o.URL,
			"headers":  o.Headers,
			"expected": o.ExpectedStatus,
			"got":      resp.StatusCode,
		}).Infof("HTTP Status could not be matched, expected %d, received %d", o.ExpectedStatus, resp.StatusCode)

	}
	return fmt.Errorf("failed to match status code: %d: %v", o.ExpectedStatus, err)
}
<commit_msg>Don't show download bar when in json mode, for php-perfect/ddev-intellij-plugin#56 (#3860)<commit_after>package util

import (
	"errors"
	"fmt"
	"golang.org/x/term"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/drud/ddev/pkg/output"
	log "github.com/sirupsen/logrus"
)

// DownloadFile retrieves a file.
func DownloadFile(destPath string, url string, progressBar bool) (err error) {
	if output.JSONOutput || !term.IsTerminal(int(os.Stdin.Fd())) {
		progressBar = false
	}
	// Create the file
	out, err := os.Create(destPath)
	if err != nil {
		return err
	}
	defer CheckClose(out)

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer CheckClose(resp.Body)

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("download link %s returned wrong status code: got %v want %v", url, resp.StatusCode, http.StatusOK)
	}
	reader := resp.Body
	if progressBar {

		bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES).Prefix(filepath.Base(destPath))
		bar.Start()

		// create proxy reader
		reader = bar.NewProxyReader(resp.Body)
		// Writer the body to file
		_, err = io.Copy(out, reader)
		bar.Finish()
	} else {
		_, err = io.Copy(out, reader)
	}

	if err != nil {
		return err
	}

	return nil
}

// HTTPOptions defines the URL and other common HTTP options for EnsureHTTPStatus.
type HTTPOptions struct {
	URL            string
	Username       string
	Password       string
	Timeout        time.Duration
	TickerInterval time.Duration
	ExpectedStatus int
	Headers        map[string]string
}

// NewHTTPOptions returns a new HTTPOptions struct with some sane defaults.
func NewHTTPOptions(URL string) *HTTPOptions {
	o := HTTPOptions{
		URL:            URL,
		TickerInterval: 20,
		Timeout:        60,
		ExpectedStatus: http.StatusOK,
		Headers:        make(map[string]string),
	}
	return &o
}

// EnsureHTTPStatus will verify a URL responds with a given response code within the Timeout period (in seconds)
func EnsureHTTPStatus(o *HTTPOptions) error {

	client := &http.Client{
		Timeout: o.Timeout * time.Second,
	}
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return errors.New("received http redirect")
	}

	req, err := http.NewRequest("GET", o.URL, nil)
	if err != nil {
		return err
	}
	if o.Username != "" && o.Password != "" {
		req.SetBasicAuth(o.Username, o.Password)
	}

	if len(o.Headers) > 0 {
		for header, value := range o.Headers {
			if header == "Host" {
				req.Host = value
				continue
			}
			req.Header.Add(header, value)
		}
	}
	// Make the request
	resp, err := client.Do(req)

	if err == nil {
		defer CheckClose(resp.Body)
		if o.ExpectedStatus != 0 && resp.StatusCode == o.ExpectedStatus {
			return nil
		}
		// Log expected vs. actual if we do not get a match.
		output.UserOut.WithFields(log.Fields{
			"URL":      o.URL,
			"headers":  o.Headers,
			"expected": o.ExpectedStatus,
			"got":      resp.StatusCode,
		}).Infof("HTTP Status could not be matched, expected %d, received %d", o.ExpectedStatus, resp.StatusCode)

	}
	return fmt.Errorf("failed to match status code: %d: %v", o.ExpectedStatus, err)
}
<|endoftext|>


<commit_before>package vfs

import (
	"image"
	"io"

	// Packages image/... are not used explicitly in the code below,
	// but are imported for its initialization side-effect
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	// Same for image/webp
	_ "golang.org/x/image/webp"

	"github.com/cozy/goexif2/exif"
)

// MetadataExtractorVersion is the version number of the metadata extractor.
// It will be used later to know which files can be re-examined to get more
// metadata when the extractor is improved.
const MetadataExtractorVersion = 1

// Metadata is a list of metadata specific to each mimetype:
// id3 for music, exif for jpegs, etc.
type Metadata map[string]interface{}

// NewMetadata returns a new metadata object, with the version field set
func NewMetadata() Metadata {
	m := Metadata{}
	m["extractor_version"] = MetadataExtractorVersion
	return m
}

// MetaExtractor is an interface for extracting metadata from a file
type MetaExtractor interface {
	io.WriteCloser
	Abort(error)
	Result() Metadata
}

// NewMetaExtractor returns an extractor for metadata if the mime type has one,
// or null else
func NewMetaExtractor(doc *FileDoc) *MetaExtractor {
	var e MetaExtractor
	switch doc.Mime {
	case "image/jpg":
		e = NewExifExtractor()
	case "image/png", "image/gif":
		e = NewImageExtractor()
	}
	if e != nil {
		return &e
	}
	return nil
}

// ImageExtractor is used to extract width/height from images
type ImageExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	ch chan interface{}
}

// NewImageExtractor returns an extractor for images
func NewImageExtractor() *ImageExtractor {
	e := &ImageExtractor{}
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ImageExtractor) Start() {
	cfg, _, err := image.DecodeConfig(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- cfg
	}
}

// Write is called to push some bytes to the extractor
func (e *ImageExtractor) Write(p []byte) (n int, err error) {
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ImageExtractor) Close() error {
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ImageExtractor) Abort(err error) {
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ImageExtractor) Result() Metadata {
	m := NewMetadata()
	cfg := <-e.ch
	switch cfg := cfg.(type) {
	case image.Config:
		m["width"] = cfg.Width
		m["height"] = cfg.Height
	}
	return m
}

// ExifExtractor is used to extract EXIF metadata from jpegs
type ExifExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	im *ImageExtractor
	ch chan interface{}
}

// NewExifExtractor returns an extractor for EXIF metadata
func NewExifExtractor() *ExifExtractor {
	e := &ExifExtractor{}
	e.im = NewImageExtractor()
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ExifExtractor) Start() {
	x, err := exif.Decode(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- x
	}
}

// Write is called to push some bytes to the extractor
func (e *ExifExtractor) Write(p []byte) (n int, err error) {
	e.im.Write(p)
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ExifExtractor) Close() error {
	e.im.Close()
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ExifExtractor) Abort(err error) {
	e.im.Abort(err)
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ExifExtractor) Result() Metadata {
	m := e.im.Result()
	x := <-e.ch
	switch x := x.(type) {
	case *exif.Exif:
		if dt, err := x.DateTime(); err == nil {
			m["datetime"] = dt
		}
		if flash, err := x.Flash(); err == nil {
			m["flash"] = flash
		}
		if lat, long, err := x.LatLong(); err == nil {
			m["gps"] = map[string]float64{
				"lat":  lat,
				"long": long,
			}
		}
	}
	return m
}
<commit_msg>Use upstream for goexif2<commit_after>package vfs

import (
	"image"
	"io"

	// Packages image/... are not used explicitly in the code below,
	// but are imported for its initialization side-effect
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	// Same for image/webp
	_ "golang.org/x/image/webp"

	"github.com/xor-gate/goexif2/exif"
)

// MetadataExtractorVersion is the version number of the metadata extractor.
// It will be used later to know which files can be re-examined to get more
// metadata when the extractor is improved.
const MetadataExtractorVersion = 1

// Metadata is a list of metadata specific to each mimetype:
// id3 for music, exif for jpegs, etc.
type Metadata map[string]interface{}

// NewMetadata returns a new metadata object, with the version field set
func NewMetadata() Metadata {
	m := Metadata{}
	m["extractor_version"] = MetadataExtractorVersion
	return m
}

// MetaExtractor is an interface for extracting metadata from a file
type MetaExtractor interface {
	io.WriteCloser
	Abort(error)
	Result() Metadata
}

// NewMetaExtractor returns an extractor for metadata if the mime type has one,
// or null else
func NewMetaExtractor(doc *FileDoc) *MetaExtractor {
	var e MetaExtractor
	switch doc.Mime {
	case "image/jpg":
		e = NewExifExtractor()
	case "image/png", "image/gif":
		e = NewImageExtractor()
	}
	if e != nil {
		return &e
	}
	return nil
}

// ImageExtractor is used to extract width/height from images
type ImageExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	ch chan interface{}
}

// NewImageExtractor returns an extractor for images
func NewImageExtractor() *ImageExtractor {
	e := &ImageExtractor{}
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ImageExtractor) Start() {
	cfg, _, err := image.DecodeConfig(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- cfg
	}
}

// Write is called to push some bytes to the extractor
func (e *ImageExtractor) Write(p []byte) (n int, err error) {
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ImageExtractor) Close() error {
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ImageExtractor) Abort(err error) {
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ImageExtractor) Result() Metadata {
	m := NewMetadata()
	cfg := <-e.ch
	switch cfg := cfg.(type) {
	case image.Config:
		m["width"] = cfg.Width
		m["height"] = cfg.Height
	}
	return m
}

// ExifExtractor is used to extract EXIF metadata from jpegs
type ExifExtractor struct {
	w  *io.PipeWriter
	r  *io.PipeReader
	im *ImageExtractor
	ch chan interface{}
}

// NewExifExtractor returns an extractor for EXIF metadata
func NewExifExtractor() *ExifExtractor {
	e := &ExifExtractor{}
	e.im = NewImageExtractor()
	e.r, e.w = io.Pipe()
	e.ch = make(chan interface{})
	go e.Start()
	return e
}

// Start is used in a goroutine to start the metadata extraction
func (e *ExifExtractor) Start() {
	x, err := exif.Decode(e.r)
	e.r.Close()
	if err != nil {
		e.ch <- err
	} else {
		e.ch <- x
	}
}

// Write is called to push some bytes to the extractor
func (e *ExifExtractor) Write(p []byte) (n int, err error) {
	e.im.Write(p)
	return e.w.Write(p)
}

// Close is called when all the bytes has been pushed, to finalize the extraction
func (e *ExifExtractor) Close() error {
	e.im.Close()
	return e.w.Close()
}

// Abort is called when the extractor can be discarded
func (e *ExifExtractor) Abort(err error) {
	e.im.Abort(err)
	e.w.CloseWithError(err)
	<-e.ch
}

// Result is called to get the extracted metadata
func (e *ExifExtractor) Result() Metadata {
	m := e.im.Result()
	x := <-e.ch
	switch x := x.(type) {
	case *exif.Exif:
		if dt, err := x.DateTime(); err == nil {
			m["datetime"] = dt
		}
		if flash, err := x.Flash(); err == nil {
			m["flash"] = flash
		}
		if lat, long, err := x.LatLong(); err == nil {
			m["gps"] = map[string]float64{
				"lat":  lat,
				"long": long,
			}
		}
	}
	return m
}
<|endoftext|>


<commit_before>package bob

import (
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
)

/*
CopyFile copies one file from source to dest.  Copied from
https://github.com/opesun/copyrecur.
*/
func CopyFile(source string, dest string) (err error) {
	sf, err := os.Open(source)
	if err != nil {
		return err
	}
	defer sf.Close()
	df, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer df.Close()
	_, err = io.Copy(df, sf)
	if err == nil {
		si, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, si.Mode())
		}
	}
	return
}

/*
CopyDir recursively copies one dir from source to dest.  Copied from
https://github.com/opesun/copyrecur.
*/
func CopyDir(source string, dest string) (err error) {
	// get properties of source dir
	fi, err := os.Stat(source)
	if err != nil {
		return err
	}
	if !fi.IsDir() {
		return errors.New("source is not a directory")
	}
	// ensure dest dir does not already exist
	_, err = os.Open(dest)
	if !os.IsNotExist(err) {
		return errors.New("destination already exists")
	}
	// create dest dir
	err = os.MkdirAll(dest, fi.Mode())
	if err != nil {
		return err
	}
	entries, err := ioutil.ReadDir(source)
	for _, entry := range entries {
		sfp := source + "/" + entry.Name()
		dfp := dest + "/" + entry.Name()
		if entry.IsDir() {
			err = CopyDir(sfp, dfp)
			if err != nil {
				log.Println(err)
			}
		} else {
			// perform copy
			err = CopyFile(sfp, dfp)
			if err != nil {
				log.Println(err)
			}
		}

	}
	return
}
<commit_msg>Refactoring the file/dir copy helpers<commit_after>package bob

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
)

/*
CopyFile copies one file from source to dest.  Copied from
https://gist.github.com/elazarl/5507969 and modified.
*/
func CopyFile(s string, d string) (err error) {
	source, err := os.Open(s)
	if err != nil {
		return
	}

	defer source.Close()

	dest, err := os.Create(d)
	if err != nil {
		return
	}

	_, err = io.Copy(dest, source)
	if err != nil {
		return
	}

	return dest.Close()
}

/*
CopyDir recursively copies one dir from source to dest.  Copied from
https://github.com/opesun/copyrecur.
*/
func CopyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceInfo, err := os.Stat(source)
	if err != nil {
		return err
	}
	if !sourceInfo.IsDir() {
		return errors.New("source is not a directory")
	}

	// ensure dest dir does not already exist
	if _, err = os.Open(dest); !os.IsNotExist(err) {
		return errors.New("destination already exists")
	}
	// create dest dir
	if err = os.MkdirAll(dest, sourceInfo.Mode()); err != nil {
		return
	}

	files, err := ioutil.ReadDir(source)

	for _, file := range files {
		sourceFilePath := source + "/" + file.Name()
		destFilePath := dest + "/" + file.Name()

		if file.IsDir() {
			if err = CopyDir(sourceFilePath, destFilePath); err != nil {
				return
			}
		} else {
			if err = CopyFile(sourceFilePath, destFilePath); err != nil {
				return
			}
		}

	}
	return
}
<|endoftext|>


<commit_before>package gbimporter

import (
	"fmt"
	"go/build"
	"go/types"
	"path/filepath"
	"strings"
	"sync"
)

// We need to mangle go/build.Default to make gcimporter work as
// intended, so use a lock to protect against concurrent accesses.
var buildDefaultLock sync.Mutex

// importer implements types.ImporterFrom and provides transparent
// support for gb-based projects.
type importer struct {
	underlying types.ImporterFrom
	ctx        *PackedContext
	gbroot     string
	gbpaths    []string
}

func New(ctx *PackedContext, filename string, underlying types.ImporterFrom) types.ImporterFrom {
	imp := &importer{
		ctx:        ctx,
		underlying: underlying,
	}

	slashed := filepath.ToSlash(filename)
	i := strings.LastIndex(slashed, "/vendor/src/")
	if i < 0 {
		i = strings.LastIndex(slashed, "/src/")
	}
	if i > 0 {
		paths := filepath.SplitList(imp.ctx.GOPATH)

		gbroot := filepath.FromSlash(slashed[:i])
		gbvendor := filepath.Join(gbroot, "vendor")
		if samePath(gbroot, imp.ctx.GOROOT) {
			goto Found
		}
		for _, path := range paths {
			if samePath(path, gbroot) || samePath(path, gbvendor) {
				goto Found
			}
		}

		imp.gbroot = gbroot
		imp.gbpaths = append(paths, gbroot, gbvendor)
	Found:
	}

	return imp
}

func (i *importer) Import(path string) (*types.Package, error) {
	return i.ImportFrom(path, "", 0)
}

func (i *importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) {
	buildDefaultLock.Lock()
	defer buildDefaultLock.Unlock()

	origDef := build.Default
	defer func() { build.Default = origDef }()

	def := &build.Default
	def.GOARCH = i.ctx.GOARCH
	def.GOOS = i.ctx.GOOS
	def.GOROOT = i.ctx.GOROOT
	def.GOPATH = i.ctx.GOPATH
	def.CgoEnabled = i.ctx.CgoEnabled
	def.UseAllFiles = i.ctx.UseAllFiles
	def.Compiler = i.ctx.Compiler
	def.BuildTags = i.ctx.BuildTags
	def.ReleaseTags = i.ctx.ReleaseTags
	def.InstallSuffix = i.ctx.InstallSuffix

	def.SplitPathList = i.splitPathList
	def.JoinPath = i.joinPath

	return i.underlying.ImportFrom(path, srcDir, mode)
}

func (i *importer) splitPathList(list string) []string {
	if i.gbroot != "" {
		return i.gbpaths
	}
	return filepath.SplitList(list)
}

func (i *importer) joinPath(elem ...string) string {
	res := filepath.Join(elem...)

	if i.gbroot != "" {
		// Want to rewrite "$GBROOT/(vendor/)?pkg/$GOOS_$GOARCH(_)?"
		// into "$GBROOT/pkg/$GOOS-$GOARCH(-)?".
		// Note: gb doesn't use vendor/pkg.
		if gbrel, err := filepath.Rel(i.gbroot, res); err == nil {
			gbrel = filepath.ToSlash(gbrel)
			gbrel, _ = match(gbrel, "vendor/")
			if gbrel, ok := match(gbrel, fmt.Sprintf("pkg/%s_%s", i.ctx.GOOS, i.ctx.GOARCH)); ok {
				gbrel, hasSuffix := match(gbrel, "_")

				// Reassemble into result.
				if hasSuffix {
					gbrel = "-" + gbrel
				}
				gbrel = fmt.Sprintf("pkg/%s-%s/", i.ctx.GOOS, i.ctx.GOARCH) + gbrel
				gbrel = filepath.FromSlash(gbrel)
				res = filepath.Join(i.gbroot, gbrel)
			}
		}
	}

	return res
}

func match(s, prefix string) (string, bool) {
	rest := strings.TrimPrefix(s, prefix)
	return rest, len(rest) < len(s)
}
<commit_msg>use source importer when default importer fails<commit_after>package gbimporter

import (
	"fmt"
	"go/build"
	goimporter "go/importer"
	"go/types"
	"path/filepath"
	"strings"
	"sync"
)

// We need to mangle go/build.Default to make gcimporter work as
// intended, so use a lock to protect against concurrent accesses.
var buildDefaultLock sync.Mutex

var srcImporter types.ImporterFrom = goimporter.For("source", nil).(types.ImporterFrom)

// importer implements types.ImporterFrom and provides transparent
// support for gb-based projects.
type importer struct {
	underlying types.ImporterFrom
	ctx        *PackedContext
	gbroot     string
	gbpaths    []string
}

func New(ctx *PackedContext, filename string, underlying types.ImporterFrom) types.ImporterFrom {
	imp := &importer{
		ctx:        ctx,
		underlying: underlying,
	}

	slashed := filepath.ToSlash(filename)
	i := strings.LastIndex(slashed, "/vendor/src/")
	if i < 0 {
		i = strings.LastIndex(slashed, "/src/")
	}
	if i > 0 {
		paths := filepath.SplitList(imp.ctx.GOPATH)

		gbroot := filepath.FromSlash(slashed[:i])
		gbvendor := filepath.Join(gbroot, "vendor")
		if samePath(gbroot, imp.ctx.GOROOT) {
			goto Found
		}
		for _, path := range paths {
			if samePath(path, gbroot) || samePath(path, gbvendor) {
				goto Found
			}
		}

		imp.gbroot = gbroot
		imp.gbpaths = append(paths, gbroot, gbvendor)
	Found:
	}

	return imp
}

func (i *importer) Import(path string) (*types.Package, error) {
	return i.ImportFrom(path, "", 0)
}

func (i *importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) {
	buildDefaultLock.Lock()
	defer buildDefaultLock.Unlock()

	origDef := build.Default
	defer func() { build.Default = origDef }()

	def := &build.Default
	def.GOARCH = i.ctx.GOARCH
	def.GOOS = i.ctx.GOOS
	def.GOROOT = i.ctx.GOROOT
	def.GOPATH = i.ctx.GOPATH
	def.CgoEnabled = i.ctx.CgoEnabled
	def.UseAllFiles = i.ctx.UseAllFiles
	def.Compiler = i.ctx.Compiler
	def.BuildTags = i.ctx.BuildTags
	def.ReleaseTags = i.ctx.ReleaseTags
	def.InstallSuffix = i.ctx.InstallSuffix

	def.SplitPathList = i.splitPathList
	def.JoinPath = i.joinPath

	pkg, err := i.underlying.ImportFrom(path, srcDir, mode)
	if pkg == nil {
		// If importing fails, try importing with source importer.
		pkg, _ = srcImporter.ImportFrom(path, srcDir, mode)
	}
	return pkg, err
}

func (i *importer) splitPathList(list string) []string {
	if i.gbroot != "" {
		return i.gbpaths
	}
	return filepath.SplitList(list)
}

func (i *importer) joinPath(elem ...string) string {
	res := filepath.Join(elem...)

	if i.gbroot != "" {
		// Want to rewrite "$GBROOT/(vendor/)?pkg/$GOOS_$GOARCH(_)?"
		// into "$GBROOT/pkg/$GOOS-$GOARCH(-)?".
		// Note: gb doesn't use vendor/pkg.
		if gbrel, err := filepath.Rel(i.gbroot, res); err == nil {
			gbrel = filepath.ToSlash(gbrel)
			gbrel, _ = match(gbrel, "vendor/")
			if gbrel, ok := match(gbrel, fmt.Sprintf("pkg/%s_%s", i.ctx.GOOS, i.ctx.GOARCH)); ok {
				gbrel, hasSuffix := match(gbrel, "_")

				// Reassemble into result.
				if hasSuffix {
					gbrel = "-" + gbrel
				}
				gbrel = fmt.Sprintf("pkg/%s-%s/", i.ctx.GOOS, i.ctx.GOARCH) + gbrel
				gbrel = filepath.FromSlash(gbrel)
				res = filepath.Join(i.gbroot, gbrel)
			}
		}
	}

	return res
}

func match(s, prefix string) (string, bool) {
	rest := strings.TrimPrefix(s, prefix)
	return rest, len(rest) < len(s)
}
<|endoftext|>


<commit_before>package bot

import "github.com/nlopes/slack"
import "strings"
import "math/rand"

func postResponse(bot *BaseBot, channel string, emoji string, name string, response string) {
	bot.PostMessage(channel, response, slack.PostMessageParameters{
		AsUser:    false,
		IconEmoji: emoji,
		Username:  name,
	})
}

func randomResponse(bot *BaseBot, channel string, emoji string, name string, responses []string) {
	response := responses[rand.Intn(len(responses))]
	postResponse(bot, channel, emoji, name, response)
}

var (
	anduinresp []string = []string{
		"안녕하세요!",
		"감사합니다!",
		"이게... 아닌데...",
		"빛이 당신을 태울 것입니다!",
		"정말 잘하셨어요.",
		"죄송합니다.",
	}
)

func specialResponses(bot *BaseBot, e *slack.MessageEvent) {
	if strings.Contains(e.Text, "72") {
		postResponse(bot, e.Channel, ":kutt:", "치하야", "큿")
	}
	if strings.Contains(e.Text, "크킄") {
		postResponse(bot, e.Channel, ":chuni:", "흑염룡", "흐콰한다")
	}
	if strings.Contains(e.Text, "안두인") {
		randomResponse(bot, e.Channel, ":anduin:", "안두인", anduinresp)
	}
}
<commit_msg>add more kutt<commit_after>package bot

import "github.com/nlopes/slack"
import "strings"
import "math/rand"

func postResponse(bot *BaseBot, channel string, emoji string, name string, response string) {
	bot.PostMessage(channel, response, slack.PostMessageParameters{
		AsUser:    false,
		IconEmoji: emoji,
		Username:  name,
	})
}

func randomResponse(bot *BaseBot, channel string, emoji string, name string, responses []string) {
	response := responses[rand.Intn(len(responses))]
	postResponse(bot, channel, emoji, name, response)
}

var (
	anduinresp []string = []string{
		"안녕하세요!",
		"감사합니다!",
		"이게... 아닌데...",
		"빛이 당신을 태울 것입니다!",
		"정말 잘하셨어요.",
		"죄송합니다.",
	}
)

func specialResponses(bot *BaseBot, e *slack.MessageEvent) {
	if strings.Contains(e.Text, "72") || strings.Contains(e.Text, "치하야") || strings.Contains(e.Text, "큿") {
		postResponse(bot, e.Channel, ":kutt:", "치하야", "큿")
	}
	if strings.Contains(e.Text, "크킄") {
		postResponse(bot, e.Channel, ":chuni:", "흑염룡", "흐콰한다")
	}
	if strings.Contains(e.Text, "안두인") {
		randomResponse(bot, e.Channel, ":anduin:", "안두인", anduinresp)
	}
}
<|endoftext|>


<commit_before>package main

import (
        "fmt"
        "net"
        "strings"
        "os"
        "bufio"
        "flag"
)

func LoadGooglebotList() []string {
        file, err := os.Open("googlebot.data")
        if err != nil {
                fmt.Println("Couldn't open googlebot.data:", err)
                os.Exit(1)
        }
        defer file.Close()

        var lines []string
        scanner := bufio.NewScanner(file)
        for scanner.Scan() {
                lines = append(lines, scanner.Text())
        }

        return lines
}

func IdentifiesAsGooglebot(list []string, useragent string) bool {
        for _, test := range list {
                if strings.Contains(test, useragent) {
                        return true
                }
        }

        return false
}

func IsGooglebot(lookupResult []string) bool {
        domain := strings.Join(strings.Split(lookupResult[0], ".")[1:3], ".")
        //      fmt.Println(domain)
        if domain == "google.com" || domain == "googlebot.com" {
                return true
        } else {
                return false
        }
}

func main() {
        addressPtr   := flag.String("address", "", "IP address of requester")
        useragentPtr := flag.String("useragent", "", "User Agent of requester")
        flag.Parse()

        list := LoadGooglebotList()

        if IdentifiesAsGooglebot(list, *useragentPtr) {
                addr, err := net.LookupAddr(*addressPtr)
                if err != nil {
                        fmt.Println(err)
                        os.Exit(1)
                }
                if IsGooglebot(addr) {
                        fmt.Println("YES")
                } else {
                        fmt.Println("NO")
                }
        }
}
<commit_msg>Extract and look through all reverse domains<commit_after>package main

import (
        "fmt"
        "net"
        "strings"
        "os"
        "bufio"
        "flag"
)

func LoadGooglebotList() []string {
        file, err := os.Open("googlebot.data")
        if err != nil {
                fmt.Println("Couldn't open googlebot.data:", err)
                os.Exit(1)
        }
        defer file.Close()

        var lines []string
        scanner := bufio.NewScanner(file)
        for scanner.Scan() {
                lines = append(lines, scanner.Text())
        }

        return lines
}

func IdentifiesAsGooglebot(list []string, useragent string) bool {
        for _, test := range list {
                if strings.Contains(test, useragent) {
                        return true
                }
        }

        return false
}

func IsGooglebot(lookupResult []string) bool {
        for _, result := range lookupResult {
                parts := strings.Split(result, ".")
                domain := strings.Join(parts[len(parts) - 3:len(parts) - 1], ".")
                fmt.Println(domain)
                if domain == "google.com" || domain == "googlebot.com" {
                        return true
		}
	}
	return false
}

func Lookup(address string, useragent string, list []string) {
        if IdentifiesAsGooglebot(list, useragent) {
                addr, err := net.LookupAddr(address)
                if err != nil {
                        fmt.Println(err)
                        os.Exit(1)
                }
                if IsGooglebot(addr) {
                        fmt.Println("YES")
                } else {
                        fmt.Println("NO")
                }
        }
}

func main() {
        addressPtr   := flag.String("address", "", "IP address of requester")
        useragentPtr := flag.String("useragent", "", "User Agent of requester")
        flag.Parse()

        list := LoadGooglebotList()

        Lookup(*addressPtr, *useragentPtr, list)
}
<|endoftext|>


<commit_before>package text

import (
	"image/color"
	"math"
	"unicode"
	"unicode/utf8"

	"github.com/faiface/pixel"
	"golang.org/x/image/font"
)

var ASCII []rune

func init() {
	ASCII = make([]rune, unicode.MaxASCII-32)
	for i := range ASCII {
		ASCII[i] = rune(32 + i)
	}
}

func RangeTable(table *unicode.RangeTable) []rune {
	var runes []rune
	for _, rng := range table.R16 {
		for r := rng.Lo; r <= rng.Hi; r += rng.Stride {
			runes = append(runes, rune(r))
		}
	}
	for _, rng := range table.R32 {
		for r := rng.Lo; r <= rng.Hi; r += rng.Stride {
			runes = append(runes, rune(r))
		}
	}
	return runes
}

type Text struct {
	Orig pixel.Vec
	Dot  pixel.Vec

	atlas *Atlas

	lineHeight float64
	tabWidth   float64

	buf    []byte
	prevR  rune
	bounds pixel.Rect
	glyph  pixel.TrianglesData
	tris   pixel.TrianglesData

	mat    pixel.Matrix
	col    pixel.RGBA
	trans  pixel.TrianglesData
	transD pixel.Drawer
	dirty  bool
}

func New(face font.Face, runeSets ...[]rune) *Text {
	runes := []rune{unicode.ReplacementChar}
	for _, set := range runeSets {
		runes = append(runes, set...)
	}

	atlas := NewAtlas(face, runes)

	txt := &Text{
		atlas:      atlas,
		lineHeight: atlas.LineHeight(),
		tabWidth:   atlas.Glyph(' ').Advance * 4,
		mat:        pixel.IM,
		col:        pixel.Alpha(1),
	}

	txt.glyph.SetLen(6)
	for i := range txt.glyph {
		txt.glyph[i].Color = pixel.Alpha(1)
		txt.glyph[i].Intensity = 1
	}

	txt.transD.Picture = txt.atlas.pic
	txt.transD.Triangles = &txt.trans

	txt.Clear()

	return txt
}

func (txt *Text) Atlas() *Atlas {
	return txt.atlas
}

func (txt *Text) SetMatrix(m pixel.Matrix) {
	if txt.mat != m {
		txt.mat = m
		txt.dirty = true
	}
}

func (txt *Text) SetColorMask(c color.Color) {
	rgba := pixel.ToRGBA(c)
	if txt.col != rgba {
		txt.col = rgba
		txt.dirty = true
	}
}

func (txt *Text) Bounds() pixel.Rect {
	return txt.bounds
}

func (txt *Text) BoundsOf(s string) pixel.Rect {
	dot := txt.Dot
	prevR := txt.prevR
	bounds := pixel.Rect{}

	for _, r := range s {
		var b pixel.Rect
		_, _, b, dot = txt.Atlas().DrawRune(prevR, r, dot)

		if bounds.W()*bounds.H() == 0 {
			bounds = b
		} else {
			bounds = bounds.Union(b)
		}

		prevR = r
	}

	return bounds
}

func (txt *Text) Color(c color.Color) {
	rgba := pixel.ToRGBA(c)
	for i := range txt.glyph {
		txt.glyph[i].Color = rgba
	}
}

func (txt *Text) LineHeight(scale float64) {
	txt.lineHeight = scale
}

func (txt *Text) TabWidth(width float64) {
	txt.tabWidth = width
}

func (txt *Text) Clear() {
	txt.prevR = -1
	txt.bounds = pixel.Rect{}
	txt.tris.SetLen(0)
	txt.dirty = true
}

func (txt *Text) Write(p []byte) (n int, err error) {
	txt.buf = append(txt.buf, p...)
	txt.drawBuf()
	return len(p), nil
}

func (txt *Text) WriteString(s string) (n int, err error) {
	txt.buf = append(txt.buf, s...)
	txt.drawBuf()
	return len(s), nil
}

func (txt *Text) WriteByte(c byte) error {
	txt.buf = append(txt.buf, c)
	txt.drawBuf()
	return nil
}

func (txt *Text) WriteRune(r rune) (n int, err error) {
	var b [4]byte
	n = utf8.EncodeRune(b[:], r)
	txt.buf = append(txt.buf, b[:n]...)
	txt.drawBuf()
	return n, nil
}

func (txt *Text) drawBuf() {
	for utf8.FullRune(txt.buf) {
		r, size := utf8.DecodeRune(txt.buf)
		txt.buf = txt.buf[size:]

		switch r {
		case '\n':
			txt.Dot -= pixel.Y(txt.lineHeight)
			txt.Dot = txt.Dot.WithX(txt.Orig.X())
			continue
		case '\r':
			txt.Dot = txt.Dot.WithX(txt.Orig.X())
			continue
		case '\t':
			rem := math.Mod(txt.Dot.X()-txt.Orig.X(), txt.tabWidth)
			rem = math.Mod(rem, rem+txt.tabWidth)
			if rem == 0 {
				rem = txt.tabWidth
			}
			txt.Dot += pixel.X(rem)
			continue
		}

		var rect, frame, bounds pixel.Rect
		rect, frame, bounds, txt.Dot = txt.Atlas().DrawRune(txt.prevR, r, txt.Dot)

		txt.prevR = r

		rv := [...]pixel.Vec{pixel.V(rect.Min.X(), rect.Min.Y()),
			pixel.V(rect.Max.X(), rect.Min.Y()),
			pixel.V(rect.Max.X(), rect.Max.Y()),
			pixel.V(rect.Min.X(), rect.Max.Y()),
		}

		fv := [...]pixel.Vec{pixel.V(frame.Min.X(), frame.Min.Y()),
			pixel.V(frame.Max.X(), frame.Min.Y()),
			pixel.V(frame.Max.X(), frame.Max.Y()),
			pixel.V(frame.Min.X(), frame.Max.Y()),
		}

		for i, j := range [...]int{0, 1, 2, 0, 2, 3} {
			txt.glyph[i].Position = rv[j]
			txt.glyph[i].Picture = fv[j]
		}

		txt.tris = append(txt.tris, txt.glyph...)
		txt.dirty = true

		if txt.bounds.W()*txt.bounds.H() == 0 {
			txt.bounds = bounds
		} else {
			txt.bounds = txt.bounds.Union(bounds)
		}
	}
}

func (txt *Text) Draw(t pixel.Target) {
	if txt.dirty {
		txt.trans.SetLen(txt.tris.Len())
		txt.trans.Update(&txt.tris)
		for i := range txt.trans {
			txt.trans[i].Position = txt.mat.Project(txt.trans[i].Position)
			txt.trans[i].Color = txt.trans[i].Color.Mul(txt.col)
		}
		txt.transD.Dirty()
		txt.dirty = false
	}
	txt.transD.Draw(t)
}
<commit_msg>improve Text code<commit_after>package text

import (
	"image/color"
	"math"
	"unicode"
	"unicode/utf8"

	"github.com/faiface/pixel"
	"golang.org/x/image/font"
)

var ASCII []rune

func init() {
	ASCII = make([]rune, unicode.MaxASCII-32)
	for i := range ASCII {
		ASCII[i] = rune(32 + i)
	}
}

func RangeTable(table *unicode.RangeTable) []rune {
	var runes []rune
	for _, rng := range table.R16 {
		for r := rng.Lo; r <= rng.Hi; r += rng.Stride {
			runes = append(runes, rune(r))
		}
	}
	for _, rng := range table.R32 {
		for r := rng.Lo; r <= rng.Hi; r += rng.Stride {
			runes = append(runes, rune(r))
		}
	}
	return runes
}

type Text struct {
	Orig pixel.Vec
	Dot  pixel.Vec

	atlas *Atlas

	lineHeight float64
	tabWidth   float64

	buf    []byte
	prevR  rune
	bounds pixel.Rect
	glyph  pixel.TrianglesData
	tris   pixel.TrianglesData

	mat    pixel.Matrix
	col    pixel.RGBA
	trans  pixel.TrianglesData
	transD pixel.Drawer
	dirty  bool
}

func New(face font.Face, runeSets ...[]rune) *Text {
	runes := []rune{unicode.ReplacementChar}
	for _, set := range runeSets {
		runes = append(runes, set...)
	}

	atlas := NewAtlas(face, runes)

	txt := &Text{
		atlas:      atlas,
		lineHeight: atlas.LineHeight(),
		tabWidth:   atlas.Glyph(' ').Advance * 4,
		mat:        pixel.IM,
		col:        pixel.Alpha(1),
	}

	txt.glyph.SetLen(6)
	for i := range txt.glyph {
		txt.glyph[i].Color = pixel.Alpha(1)
		txt.glyph[i].Intensity = 1
	}

	txt.transD.Picture = txt.atlas.pic
	txt.transD.Triangles = &txt.trans

	txt.Clear()

	return txt
}

func (txt *Text) Atlas() *Atlas {
	return txt.atlas
}

func (txt *Text) SetMatrix(m pixel.Matrix) {
	if txt.mat != m {
		txt.mat = m
		txt.dirty = true
	}
}

func (txt *Text) SetColorMask(c color.Color) {
	rgba := pixel.ToRGBA(c)
	if txt.col != rgba {
		txt.col = rgba
		txt.dirty = true
	}
}

func (txt *Text) Bounds() pixel.Rect {
	return txt.bounds
}

func (txt *Text) BoundsOf(s string) pixel.Rect {
	dot := txt.Dot
	prevR := txt.prevR
	bounds := pixel.Rect{}

	for _, r := range s {
		var control bool
		dot, control = txt.controlRune(r, dot)
		if control {
			continue
		}

		var b pixel.Rect
		_, _, b, dot = txt.Atlas().DrawRune(prevR, r, dot)

		if bounds.W()*bounds.H() == 0 {
			bounds = b
		} else {
			bounds = bounds.Union(b)
		}

		prevR = r
	}

	return bounds
}

func (txt *Text) Color(c color.Color) {
	rgba := pixel.ToRGBA(c)
	for i := range txt.glyph {
		txt.glyph[i].Color = rgba
	}
}

func (txt *Text) LineHeight(scale float64) {
	txt.lineHeight = scale
}

func (txt *Text) TabWidth(width float64) {
	txt.tabWidth = width
}

func (txt *Text) Clear() {
	txt.prevR = -1
	txt.bounds = pixel.Rect{}
	txt.tris.SetLen(0)
	txt.dirty = true
}

func (txt *Text) Write(p []byte) (n int, err error) {
	txt.buf = append(txt.buf, p...)
	txt.drawBuf()
	return len(p), nil
}

func (txt *Text) WriteString(s string) (n int, err error) {
	txt.buf = append(txt.buf, s...)
	txt.drawBuf()
	return len(s), nil
}

func (txt *Text) WriteByte(c byte) error {
	txt.buf = append(txt.buf, c)
	txt.drawBuf()
	return nil
}

func (txt *Text) WriteRune(r rune) (n int, err error) {
	var b [4]byte
	n = utf8.EncodeRune(b[:], r)
	txt.buf = append(txt.buf, b[:n]...)
	txt.drawBuf()
	return n, nil
}

// controlRune checks if r is a control rune (newline, tab, ...). If it is, a new dot position and
// true is returned. If r is not a control rune, the original dot and false is returned.
func (txt *Text) controlRune(r rune, dot pixel.Vec) (newDot pixel.Vec, control bool) {
	switch r {
	case '\n':
		dot -= pixel.Y(txt.lineHeight)
		dot = dot.WithX(txt.Orig.X())
	case '\r':
		dot = dot.WithX(txt.Orig.X())
	case '\t':
		rem := math.Mod(dot.X()-txt.Orig.X(), txt.tabWidth)
		rem = math.Mod(rem, rem+txt.tabWidth)
		if rem == 0 {
			rem = txt.tabWidth
		}
		dot += pixel.X(rem)
	default:
		return dot, false
	}
	return dot, true
}

func (txt *Text) drawBuf() {
	for utf8.FullRune(txt.buf) {
		r, size := utf8.DecodeRune(txt.buf)
		txt.buf = txt.buf[size:]

		var control bool
		txt.Dot, control = txt.controlRune(r, txt.Dot)
		if control {
			continue
		}

		var rect, frame, bounds pixel.Rect
		rect, frame, bounds, txt.Dot = txt.Atlas().DrawRune(txt.prevR, r, txt.Dot)

		txt.prevR = r

		rv := [...]pixel.Vec{pixel.V(rect.Min.X(), rect.Min.Y()),
			pixel.V(rect.Max.X(), rect.Min.Y()),
			pixel.V(rect.Max.X(), rect.Max.Y()),
			pixel.V(rect.Min.X(), rect.Max.Y()),
		}

		fv := [...]pixel.Vec{pixel.V(frame.Min.X(), frame.Min.Y()),
			pixel.V(frame.Max.X(), frame.Min.Y()),
			pixel.V(frame.Max.X(), frame.Max.Y()),
			pixel.V(frame.Min.X(), frame.Max.Y()),
		}

		for i, j := range [...]int{0, 1, 2, 0, 2, 3} {
			txt.glyph[i].Position = rv[j]
			txt.glyph[i].Picture = fv[j]
		}

		txt.tris = append(txt.tris, txt.glyph...)
		txt.dirty = true

		if txt.bounds.W()*txt.bounds.H() == 0 {
			txt.bounds = bounds
		} else {
			txt.bounds = txt.bounds.Union(bounds)
		}
	}
}

func (txt *Text) Draw(t pixel.Target) {
	if txt.dirty {
		txt.trans.SetLen(txt.tris.Len())
		txt.trans.Update(&txt.tris)
		for i := range txt.trans {
			txt.trans[i].Position = txt.mat.Project(txt.trans[i].Position)
			txt.trans[i].Color = txt.trans[i].Color.Mul(txt.col)
		}
		txt.transD.Dirty()
		txt.dirty = false
	}
	txt.transD.Draw(t)
}
<|endoftext|>


<commit_before>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiUrlPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiUrlPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for len(ids) > 0 {
		var tIds []uint
		if len(ids) > 100 {
			tIds = ids[:100]
			ids = ids[100:]
		} else {
			tIds = ids
			ids = nil
		}
		var messageIds string
		for _, id := range tIds {
			if len(messageIds) > 0 {
				messageIds += ","
			}
			messageIds += strconv.Itoa(int(id))
		}
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<commit_msg>Added joinUints func<commit_after>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiUrlPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiUrlPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for len(ids) > 0 {
		var tIds []uint
		if len(ids) > 100 {
			tIds = ids[:100]
			ids = ids[100:]
		} else {
			tIds = ids
			ids = nil
		}
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<|endoftext|>


<commit_before>package github

import (
	"net/http"
	"sync"

	"github.com/dgrijalva/jwt-go"
	"github.com/google/go-github/github"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/jwt/provider"
	"github.com/pkg/errors"
)

func init() {
	provider.Register("github", &Provider{})
}

// Provider abstracts the authentication for github
type Provider struct {
	provider.Verifier

	teams  []Team
	config config.Credentials
}

// Build acts like the constructor for a provider
func (gp *Provider) Build(config config.Credentials) provider.Provider {
	client := NewClient()
	teams := gp.teamConfigsToTeam(config.Github.Teams)

	return &Provider{
		Verifier: provider.NewVerifierBasket(
			NewTeamVerifier(teams, client),
			NewOrganizationVerifier(config.Github.Organizations, client),
		),
		teams:  teams,
		config: config,
	}
}

// GetClaims returns a JWT Map Claim
func (gp *Provider) GetClaims(httpClient *http.Client) (jwt.MapClaims, error) {
	client := NewClient()

	var (
		wg            sync.WaitGroup
		user          *github.User
		usersOrgTeams OrganizationTeams
		err           error
	)

	wg.Add(2)

	go func() {
		defer wg.Done()
		res, err := client.CurrentUser(httpClient)
		if err != nil {
			err = append(errs, errors.Wrap(err, "failed to get github users"))
			return
		}
		user = res
	}()

	go func() {
		defer wg.Done()
		res, err := client.Teams(httpClient)
		if err != nil {
			err = append(errs, errors.Wrap(err, "failed to get github teams"))
			return
		}
		usersOrgTeams = res
	}()

	wg.Wait()

	if err != nil {
		return nil, err
	}

	return jwt.MapClaims{
		"sub":      *user.Login,
		"is_admin": gp.isAdmin(usersOrgTeams),
	}, nil
}

func (gp *Provider) teamConfigsToTeam(configTeams map[string]string) []Team {
	teams := []Team{}
	for org, team := range configTeams {
		teams = append(teams, Team{
			Name:         team,
			Organization: org,
		})
	}
	return teams
}

func (gp *Provider) isAdmin(usersOrgTeams OrganizationTeams) bool {
	for _, team := range gp.teams {
		if teams, ok := usersOrgTeams[team.Organization]; ok {
			for _, teamUserBelongsTo := range teams {
				if teamUserBelongsTo == gp.config.JanusAdminTeam {
					return true
				}
			}
		}
	}
	return false
}
<commit_msg>Fix typo<commit_after>package github

import (
	"net/http"
	"sync"

	"github.com/dgrijalva/jwt-go"
	"github.com/google/go-github/github"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/jwt/provider"
	"github.com/pkg/errors"
)

func init() {
	provider.Register("github", &Provider{})
}

// Provider abstracts the authentication for github
type Provider struct {
	provider.Verifier

	teams  []Team
	config config.Credentials
}

// Build acts like the constructor for a provider
func (gp *Provider) Build(config config.Credentials) provider.Provider {
	client := NewClient()
	teams := gp.teamConfigsToTeam(config.Github.Teams)

	return &Provider{
		Verifier: provider.NewVerifierBasket(
			NewTeamVerifier(teams, client),
			NewOrganizationVerifier(config.Github.Organizations, client),
		),
		teams:  teams,
		config: config,
	}
}

// GetClaims returns a JWT Map Claim
func (gp *Provider) GetClaims(httpClient *http.Client) (jwt.MapClaims, error) {
	client := NewClient()

	var (
		wg            sync.WaitGroup
		user          *github.User
		usersOrgTeams OrganizationTeams
		err           error
	)

	wg.Add(2)

	go func() {
		defer wg.Done()
		user, err = client.CurrentUser(httpClient)
		if err != nil {
			err = errors.Wrap(err, "failed to get github users")
		}
	}()

	go func() {
		defer wg.Done()
		usersOrgTeams, err = client.Teams(httpClient)
		if err != nil {
			err = errors.Wrap(err, "failed to get github teams")
		}
	}()

	wg.Wait()

	if err != nil {
		return nil, err
	}

	return jwt.MapClaims{
		"sub":      *user.Login,
		"is_admin": gp.isAdmin(usersOrgTeams),
	}, nil
}

func (gp *Provider) teamConfigsToTeam(configTeams map[string]string) []Team {
	teams := []Team{}
	for org, team := range configTeams {
		teams = append(teams, Team{
			Name:         team,
			Organization: org,
		})
	}
	return teams
}

func (gp *Provider) isAdmin(usersOrgTeams OrganizationTeams) bool {
	for _, team := range gp.teams {
		if teams, ok := usersOrgTeams[team.Organization]; ok {
			for _, teamUserBelongsTo := range teams {
				if teamUserBelongsTo == gp.config.JanusAdminTeam {
					return true
				}
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapper

import (
	lru "github.com/hashicorp/golang-lru"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	cacheLength = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "statsd_exporter_cache_length",
			Help: "The count of unique metrics currently cached.",
		},
	)
	cacheGetsTotal = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "mapping_cache_gets_total",
			Help: "The count of total cache gets.",
		},
	)
	cacheHitsTotal = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "mapping_cache_hits_total",
			Help: "The count of total cache hits.",
		},
	)
)

type MetricMapperCacheResult struct {
	Mapping *MetricMapping
	Matched bool
	Labels  prometheus.Labels
}

type MetricMapperCache interface {
	Get(metricString string, metricType MetricType) (*MetricMapperCacheResult, bool)
	AddMatch(metricString string, metricType MetricType, mapping *MetricMapping, labels prometheus.Labels)
	AddMiss(metricString string, metricType MetricType)
}

type MetricMapperLRUCache struct {
	MetricMapperCache
	cache *lru.Cache
}

type MetricMapperNoopCache struct {
	MetricMapperCache
}

func NewMetricMapperCache(size int) (*MetricMapperLRUCache, error) {
	cacheLength.Set(0)
	cache, err := lru.New(size)
	if err != nil {
		return &MetricMapperLRUCache{}, err
	}
	return &MetricMapperLRUCache{cache: cache}, nil
}

func (m *MetricMapperLRUCache) Get(metricString string, metricType MetricType) (*MetricMapperCacheResult, bool) {
	cacheGetsTotal.Inc()
	if result, ok := m.cache.Get(formatKey(metricString, metricType)); ok {
		cacheHitsTotal.Inc()
		return result.(*MetricMapperCacheResult), true
	} else {
		return nil, false
	}
}

func (m *MetricMapperLRUCache) AddMatch(metricString string, metricType MetricType, mapping *MetricMapping, labels prometheus.Labels) {
	go m.trackCacheLength()
	m.cache.Add(formatKey(metricString, metricType), &MetricMapperCacheResult{Mapping: mapping, Matched: true, Labels: labels})
}

func (m *MetricMapperLRUCache) AddMiss(metricString string, metricType MetricType) {
	go m.trackCacheLength()
	m.cache.Add(formatKey(metricString, metricType), &MetricMapperCacheResult{Matched: false})
}

func (m *MetricMapperLRUCache) trackCacheLength() {
	cacheLength.Set(float64(m.cache.Len()))
}

func formatKey(metricString string, metricType MetricType) string {
	return string(metricType) + "." + metricString
}

func NewMetricMapperNoopCache() *MetricMapperNoopCache {
	cacheLength.Set(0)
	return &MetricMapperNoopCache{}
}

func (m *MetricMapperNoopCache) Get(metricString string, metricType MetricType) (*MetricMapperCacheResult, bool) {
	return nil, false
}

func (m *MetricMapperNoopCache) AddMatch(metricString string, metricType MetricType, mapping *MetricMapping, labels prometheus.Labels) {
	return
}

func (m *MetricMapperNoopCache) AddMiss(metricString string, metricType MetricType) {
	return
}

func init() {
	prometheus.MustRegister(cacheLength)
	prometheus.MustRegister(cacheGetsTotal)
	prometheus.MustRegister(cacheHitsTotal)
}
<commit_msg>Rename mapping cache length metric<commit_after>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapper

import (
	lru "github.com/hashicorp/golang-lru"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	cacheLength = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "mapping_cache_length",
			Help: "The count of unique metrics currently cached.",
		},
	)
	cacheGetsTotal = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "mapping_cache_gets_total",
			Help: "The count of total cache gets.",
		},
	)
	cacheHitsTotal = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "mapping_cache_hits_total",
			Help: "The count of total cache hits.",
		},
	)
)

type MetricMapperCacheResult struct {
	Mapping *MetricMapping
	Matched bool
	Labels  prometheus.Labels
}

type MetricMapperCache interface {
	Get(metricString string, metricType MetricType) (*MetricMapperCacheResult, bool)
	AddMatch(metricString string, metricType MetricType, mapping *MetricMapping, labels prometheus.Labels)
	AddMiss(metricString string, metricType MetricType)
}

type MetricMapperLRUCache struct {
	MetricMapperCache
	cache *lru.Cache
}

type MetricMapperNoopCache struct {
	MetricMapperCache
}

func NewMetricMapperCache(size int) (*MetricMapperLRUCache, error) {
	cacheLength.Set(0)
	cache, err := lru.New(size)
	if err != nil {
		return &MetricMapperLRUCache{}, err
	}
	return &MetricMapperLRUCache{cache: cache}, nil
}

func (m *MetricMapperLRUCache) Get(metricString string, metricType MetricType) (*MetricMapperCacheResult, bool) {
	cacheGetsTotal.Inc()
	if result, ok := m.cache.Get(formatKey(metricString, metricType)); ok {
		cacheHitsTotal.Inc()
		return result.(*MetricMapperCacheResult), true
	} else {
		return nil, false
	}
}

func (m *MetricMapperLRUCache) AddMatch(metricString string, metricType MetricType, mapping *MetricMapping, labels prometheus.Labels) {
	go m.trackCacheLength()
	m.cache.Add(formatKey(metricString, metricType), &MetricMapperCacheResult{Mapping: mapping, Matched: true, Labels: labels})
}

func (m *MetricMapperLRUCache) AddMiss(metricString string, metricType MetricType) {
	go m.trackCacheLength()
	m.cache.Add(formatKey(metricString, metricType), &MetricMapperCacheResult{Matched: false})
}

func (m *MetricMapperLRUCache) trackCacheLength() {
	cacheLength.Set(float64(m.cache.Len()))
}

func formatKey(metricString string, metricType MetricType) string {
	return string(metricType) + "." + metricString
}

func NewMetricMapperNoopCache() *MetricMapperNoopCache {
	cacheLength.Set(0)
	return &MetricMapperNoopCache{}
}

func (m *MetricMapperNoopCache) Get(metricString string, metricType MetricType) (*MetricMapperCacheResult, bool) {
	return nil, false
}

func (m *MetricMapperNoopCache) AddMatch(metricString string, metricType MetricType, mapping *MetricMapping, labels prometheus.Labels) {
	return
}

func (m *MetricMapperNoopCache) AddMiss(metricString string, metricType MetricType) {
	return
}

func init() {
	prometheus.MustRegister(cacheLength)
	prometheus.MustRegister(cacheGetsTotal)
	prometheus.MustRegister(cacheHitsTotal)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbus

import (
	"fmt"

	godbus "github.com/godbus/dbus"
)

// DBusFake is a simple fake Interface type.
type DBusFake struct {
	systemBus  *DBusFakeConnection
	sessionBus *DBusFakeConnection
}

// DBusFakeConnection represents a fake D-Bus connection
type DBusFakeConnection struct {
	busObject      *fakeObject
	objects        map[string]*fakeObject
	signalHandlers []chan<- *godbus.Signal
}

// DBusFakeHandler is used to handle fake D-Bus method calls
type DBusFakeHandler func(method string, args ...interface{}) ([]interface{}, error)

type fakeObject struct {
	handler DBusFakeHandler
}

type fakeCall struct {
	ret []interface{}
	err error
}

// NewFake returns a new Interface which will fake talking to D-Bus
func NewFake(systemBus *DBusFakeConnection, sessionBus *DBusFakeConnection) *DBusFake {
	return &DBusFake{systemBus, sessionBus}
}

func NewFakeConnection() *DBusFakeConnection {
	return &DBusFakeConnection{
		objects: make(map[string]*fakeObject),
	}
}

// SystemBus is part of Interface
func (db *DBusFake) SystemBus() (Connection, error) {
	if db.systemBus != nil {
		return db.systemBus, nil
	} else {
		return nil, fmt.Errorf("DBus is not running")
	}
}

// SessionBus is part of Interface
func (db *DBusFake) SessionBus() (Connection, error) {
	if db.sessionBus != nil {
		return db.sessionBus, nil
	} else {
		return nil, fmt.Errorf("DBus is not running")
	}
}

// BusObject is part of the Connection interface
func (conn *DBusFakeConnection) BusObject() Object {
	return conn.busObject
}

// Object is part of the Connection interface
func (conn *DBusFakeConnection) Object(name, path string) Object {
	return conn.objects[name+path]
}

// Signal is part of the Connection interface
func (conn *DBusFakeConnection) Signal(ch chan<- *godbus.Signal) {
	for i := range conn.signalHandlers {
		if conn.signalHandlers[i] == ch {
			conn.signalHandlers = append(conn.signalHandlers[:i], conn.signalHandlers[i+1:]...)
			return
		}
	}
	conn.signalHandlers = append(conn.signalHandlers, ch)
}

// SetBusObject sets the handler for the BusObject of conn
func (conn *DBusFakeConnection) SetBusObject(handler DBusFakeHandler) {
	conn.busObject = &fakeObject{handler}
}

// AddObject adds a handler for the Object at name and path
func (conn *DBusFakeConnection) AddObject(name, path string, handler DBusFakeHandler) {
	conn.objects[name+path] = &fakeObject{handler}
}

// EmitSignal emits a signal on conn
func (conn *DBusFakeConnection) EmitSignal(name, path, iface, signal string, args ...interface{}) {
	sig := &godbus.Signal{
		Sender: name,
		Path:   godbus.ObjectPath(path),
		Name:   iface + "." + signal,
		Body:   args,
	}
	for _, ch := range conn.signalHandlers {
		ch <- sig
	}
}

// Call is part of the Object interface
func (obj *fakeObject) Call(method string, flags godbus.Flags, args ...interface{}) Call {
	ret, err := obj.handler(method, args...)
	return &fakeCall{ret, err}
}

// Store is part of the Call interface
func (call *fakeCall) Store(retvalues ...interface{}) error {
	if call.err != nil {
		return call.err
	}
	return godbus.Store(call.ret, retvalues...)
}
<commit_msg>The DBusFakeConnection should have locks to prevent races<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbus

import (
	"fmt"
	"sync"

	godbus "github.com/godbus/dbus"
)

// DBusFake is a simple fake Interface type.
type DBusFake struct {
	systemBus  *DBusFakeConnection
	sessionBus *DBusFakeConnection
}

// DBusFakeConnection represents a fake D-Bus connection
type DBusFakeConnection struct {
	lock           sync.Mutex
	busObject      *fakeObject
	objects        map[string]*fakeObject
	signalHandlers []chan<- *godbus.Signal
}

// DBusFakeHandler is used to handle fake D-Bus method calls
type DBusFakeHandler func(method string, args ...interface{}) ([]interface{}, error)

type fakeObject struct {
	handler DBusFakeHandler
}

type fakeCall struct {
	ret []interface{}
	err error
}

// NewFake returns a new Interface which will fake talking to D-Bus
func NewFake(systemBus *DBusFakeConnection, sessionBus *DBusFakeConnection) *DBusFake {
	return &DBusFake{systemBus, sessionBus}
}

func NewFakeConnection() *DBusFakeConnection {
	return &DBusFakeConnection{
		objects: make(map[string]*fakeObject),
	}
}

// SystemBus is part of Interface
func (db *DBusFake) SystemBus() (Connection, error) {
	if db.systemBus != nil {
		return db.systemBus, nil
	} else {
		return nil, fmt.Errorf("DBus is not running")
	}
}

// SessionBus is part of Interface
func (db *DBusFake) SessionBus() (Connection, error) {
	if db.sessionBus != nil {
		return db.sessionBus, nil
	} else {
		return nil, fmt.Errorf("DBus is not running")
	}
}

// BusObject is part of the Connection interface
func (conn *DBusFakeConnection) BusObject() Object {
	return conn.busObject
}

// Object is part of the Connection interface
func (conn *DBusFakeConnection) Object(name, path string) Object {
	return conn.objects[name+path]
}

// Signal is part of the Connection interface
func (conn *DBusFakeConnection) Signal(ch chan<- *godbus.Signal) {
	conn.lock.Lock()
	defer conn.lock.Unlock()
	for i := range conn.signalHandlers {
		if conn.signalHandlers[i] == ch {
			conn.signalHandlers = append(conn.signalHandlers[:i], conn.signalHandlers[i+1:]...)
			return
		}
	}
	conn.signalHandlers = append(conn.signalHandlers, ch)
}

// SetBusObject sets the handler for the BusObject of conn
func (conn *DBusFakeConnection) SetBusObject(handler DBusFakeHandler) {
	conn.busObject = &fakeObject{handler}
}

// AddObject adds a handler for the Object at name and path
func (conn *DBusFakeConnection) AddObject(name, path string, handler DBusFakeHandler) {
	conn.objects[name+path] = &fakeObject{handler}
}

// EmitSignal emits a signal on conn
func (conn *DBusFakeConnection) EmitSignal(name, path, iface, signal string, args ...interface{}) {
	conn.lock.Lock()
	defer conn.lock.Unlock()
	sig := &godbus.Signal{
		Sender: name,
		Path:   godbus.ObjectPath(path),
		Name:   iface + "." + signal,
		Body:   args,
	}
	for _, ch := range conn.signalHandlers {
		ch <- sig
	}
}

// Call is part of the Object interface
func (obj *fakeObject) Call(method string, flags godbus.Flags, args ...interface{}) Call {
	ret, err := obj.handler(method, args...)
	return &fakeCall{ret, err}
}

// Store is part of the Call interface
func (call *fakeCall) Store(retvalues ...interface{}) error {
	if call.err != nil {
		return call.err
	}
	return godbus.Store(call.ret, retvalues...)
}
<|endoftext|>


<commit_before>package json

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/str1ngs/util/file"
	"io"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"
)

var (
	client = new(http.Client)
)

// Marshal's a interface, and writes it to a gzipped file.
func WriteGz(v interface{}, file string) (err error) {
	fd, err := os.Create(file)
	if err != nil {
		return err
	}
	defer fd.Close()
	return WriteGzIo(v, fd)
}

// Read a gzipped json file and decodes it into an interface.
func ReadGz(v interface{}, file string) (err error) {
	fd, err := os.Open(file)
	if err != nil {
		return err
	}
	defer fd.Close()
	return ReadGzIo(v, fd)
}

func ReadGzIo(v interface{}, r io.Reader) (err error) {
	gz, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	defer gz.Close()
	return json.NewDecoder(gz).Decode(v)
}

func WriteGzIo(v interface{}, w io.Writer) (err error) {
	gz := gzip.NewWriter(w)
	defer gz.Close()
	b, err := json.Marshal(v)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	err = json.Indent(buf, b, "", "\t")
	if err != nil {
		return err
	}
	_, err = io.Copy(gz, buf)
	return err
}

// Write marshals a interface and writes it to a file
func Write(v interface{}, path string) (err error) {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}
	defer fd.Close()
	return WritePretty(v, fd)
}

// Reads a json file and decodes it into interface
func Read(v interface{}, path string) (err error) {
	if !file.Exists(path) {
		return fmt.Errorf("%s does not exist.", path)
	}
	fd, err := os.Open(path)
	if err != nil {
		return err
	}
	defer fd.Close()
	return json.NewDecoder(fd).Decode(v)
}

func Clean(v interface{}, w io.Writer) (err error) {
	buf := new(bytes.Buffer)
	err = WritePretty(v, buf)
	if err != nil {
		return err
	}
	for {
		b, err := buf.ReadByte()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		switch b {
		case '{', '}', '"', ',', '[', ']', '\t':
			continue
		default:
			w.Write([]byte{b})
		}
	}
	return nil
}

func Format(in io.Reader, out io.Writer) error {
	var v interface{}
	err := json.NewDecoder(in).Decode(&v)
	if err != nil {
		return err
	}
	return WritePretty(v, out)
}

// PrintPretty marshal's an interface and ouputs formatted json to writer.
func WritePretty(v interface{}, w io.Writer) (err error) {
	b, err := json.Marshal(v)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	err = json.Indent(buf, b, "", "\t")
	//err = json.Compact(buf, b)
	if err != nil {
		return err
	}
	br := bufio.NewReader(buf)
	tw := tabwriter.NewWriter(w, 4, 0, 1, ' ', 0)
	for {
		b, _, err := br.ReadLine()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		line := string(b) + "\n"
		line = strings.Replace(line, "\":", "\"\t:", 1)
		line = strings.Replace(line, "],", "],\t", 1)
		line = strings.Replace(line, "},", "},\t", 1)
		_, err = tw.Write([]byte(line))
		if err != nil {
			return err
		}
	}
	return tw.Flush()
}

// Decodes a URL to interface
func Get(v interface{}, url string) (err error) {
	resp, err := client.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New(resp.Status)
	}
	return json.NewDecoder(resp.Body).Decode(v)
}
<commit_msg>dont leave trail tab after right brakets.<commit_after>package json

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/str1ngs/util/file"
	"io"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"
)

var (
	client = new(http.Client)
)

// Marshal's a interface, and writes it to a gzipped file.
func WriteGz(v interface{}, file string) (err error) {
	fd, err := os.Create(file)
	if err != nil {
		return err
	}
	defer fd.Close()
	return WriteGzIo(v, fd)
}

// Read a gzipped json file and decodes it into an interface.
func ReadGz(v interface{}, file string) (err error) {
	fd, err := os.Open(file)
	if err != nil {
		return err
	}
	defer fd.Close()
	return ReadGzIo(v, fd)
}

func ReadGzIo(v interface{}, r io.Reader) (err error) {
	gz, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	defer gz.Close()
	return json.NewDecoder(gz).Decode(v)
}

func WriteGzIo(v interface{}, w io.Writer) (err error) {
	gz := gzip.NewWriter(w)
	defer gz.Close()
	b, err := json.Marshal(v)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	err = json.Indent(buf, b, "", "\t")
	if err != nil {
		return err
	}
	_, err = io.Copy(gz, buf)
	return err
}

// Write marshals a interface and writes it to a file
func Write(v interface{}, path string) (err error) {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}
	defer fd.Close()
	return WritePretty(v, fd)
}

// Reads a json file and decodes it into interface
func Read(v interface{}, path string) (err error) {
	if !file.Exists(path) {
		return fmt.Errorf("%s does not exist.", path)
	}
	fd, err := os.Open(path)
	if err != nil {
		return err
	}
	defer fd.Close()
	return json.NewDecoder(fd).Decode(v)
}

func Clean(v interface{}, w io.Writer) (err error) {
	buf := new(bytes.Buffer)
	err = WritePretty(v, buf)
	if err != nil {
		return err
	}
	for {
		b, err := buf.ReadByte()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		switch b {
		case '{', '}', '"', ',', '[', ']', '\t':
			continue
		default:
			w.Write([]byte{b})
		}
	}
	return nil
}

func Format(in io.Reader, out io.Writer) error {
	var v interface{}
	err := json.NewDecoder(in).Decode(&v)
	if err != nil {
		return err
	}
	return WritePretty(v, out)
}

// PrintPretty marshal's an interface and ouputs formatted json to writer.
func WritePretty(v interface{}, w io.Writer) (err error) {
	b, err := json.Marshal(v)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	err = json.Indent(buf, b, "", "\t")
	//err = json.Compact(buf, b)
	if err != nil {
		return err
	}
	br := bufio.NewReader(buf)
	tw := tabwriter.NewWriter(w, 4, 0, 1, ' ', 0)
	for {
		b, _, err := br.ReadLine()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		line := string(b) + "\n"
		line = strings.Replace(line, "\":", "\"\t:", 1)
		line = strings.Replace(line, "},", "},\t", 1)
		_, err = tw.Write([]byte(line))
		if err != nil {
			return err
		}
	}
	return tw.Flush()
}

// Decodes a URL to interface
func Get(v interface{}, url string) (err error) {
	resp, err := client.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New(resp.Status)
	}
	return json.NewDecoder(resp.Body).Decode(v)
}
<|endoftext|>