<commit_before>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package osutil

import (
	"bytes"
	"crypto/md5"
	"io"
	"os"
)

// CopyFile tries to copy src to dst. If dst already exists, it will be
// overwritten. If it does not exist, it will be created.
func CopyFile(src, dst string) (err error) {
	// Make sure that both files are regular.
	if _, err = FileExists(src); err != nil {
		return
	}
	if _, err = FileExists(dst); err != nil {
		return
	}

	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

// MoveFile tries to move src to dst. If dst already exists, it will be
// overwritten.
func MoveFile(src, dst string) error {
	// Make sure that both files are regular.
	if _, err := FileExists(src); err != nil {
		return err
	}
	if _, err := FileExists(dst); err != nil {
		return err
	}

	err := os.Rename(src, dst)
	if err != nil {
		err := CopyFile(src, dst)
		if err != nil {
			return err
		}
		return os.Remove(src)
	}

	return nil
}

// MoveFileLazy is the same as MoveFile, except that it avoids copying
// the file when the destination already has the same contents.
func MoveFileLazy(src, dst string) error {
	same, err := SameContents(src, dst)
	if err != nil {
		return err
	}
	if same {
		return os.Remove(src)
	}
	return MoveFile(src, dst)
}

// CopyFileLazy is the same as CopyFile, except that it avoids copying
// the file when the destination already has the same contents.
func CopyFileLazy(src, dst string) error {
	same, err := SameContents(src, dst)
	if err != nil {
		return err
	}
	if same {
		return nil
	}
	return CopyFile(src, dst)
}

// SameContents returns same = true if src and dst both exist and have the
// same file contents. Whether the file data is at the same place on
// disk is a different question, which is not answered.
//
// If either file is a directory, FileTypeError is returned.
func SameContents(src, dst string) (same bool, err error) {
	same, err = SameFile(src, dst)
	if err != nil {
		return same, err
	}

	// TODO: I could make this more efficient, based on the file size.
	ssum, err := sumFile(src)
	if err != nil {
		return false, err
	}
	dsum, err := sumFile(dst)
	if err != nil {
		return false, err
	}
	return bytes.Compare(ssum, dsum) == 0, nil
}

// sumFile creates an md5sum of a file
func sumFile(path string) (sum []byte, err error) {
	h := md5.New()
	f, err := os.Open(path)
	if err != nil {
		return sum, err
	}
	io.Copy(h, f)
	return h.Sum(nil), nil
}

func SameFile(src, dst string) (same bool, err error) {
	fs, err := os.Stat(src)
	if err != nil {
		return false, err
	}
	if fs.IsDir() {
		return false, FileTypeError{src}
	}

	fd, err := os.Stat(dst)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	if fd.IsDir() {
		return false, FileTypeError{dst}
	}

	return os.SameFile(fs, fd), nil
}

// Exists returns ex = true if the given file exists, regardless whether
// it is a file or a directory. Normally you will probably want to use the more
// specific versions: FileExists and DirectoryExists.
func Exists(path string) (ex bool, err error) {
	ex, _, err = exists(path)
	return ex, err
}

// FileExists returns ex = true if the file exists and is not
// a directory, and returns err != nil if any other error occured (such as
// permission denied).
func FileExists(path string) (ex bool, err error) {
	var stat os.FileInfo

	ex, stat, err = exists(path)
	if err != nil && stat.IsDir() {
		err = FileTypeError{path}
	}
	return ex, err
}

// DirExists returns ex = true if the file exists and is
// a directory, and returns err != nil if any other error occured (such as
// permission denied).
func DirExists(path string) (ex bool, err error) {
	var stat os.FileInfo

	ex, stat, err = exists(path)
	if err != nil && !stat.IsDir() {
		err = FileTypeError{path}
	}
	return ex, err
}

// exists does the hard work for Exists, FileExists, and DirExists,
// returning ex = true if the file given by path exists.
func exists(path string) (ex bool, stat os.FileInfo, err error) {
	stat, err = os.Stat(path)

	ex = true
	if err != nil {
		// ex = true if file exists
		ex = !os.IsNotExist(err)
		if !ex {
			err = nil
		}
	}
	return ex, stat, err
}
<commit_msg>Bugfix: FileExists and DirExists nil pointer exception<commit_after>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package osutil

import (
	"bytes"
	"crypto/md5"
	"io"
	"os"
)

// CopyFile tries to copy src to dst. If dst already exists, it will be
// overwritten. If it does not exist, it will be created.
func CopyFile(src, dst string) (err error) {
	// Make sure that both files are regular.
	if _, err = FileExists(src); err != nil {
		return
	}
	if _, err = FileExists(dst); err != nil {
		return
	}

	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

// MoveFile tries to move src to dst. If dst already exists, it will be
// overwritten.
func MoveFile(src, dst string) error {
	// Make sure that both files are regular.
	if _, err := FileExists(src); err != nil {
		return err
	}
	if _, err := FileExists(dst); err != nil {
		return err
	}

	err := os.Rename(src, dst)
	if err != nil {
		err := CopyFile(src, dst)
		if err != nil {
			return err
		}
		return os.Remove(src)
	}

	return nil
}

// MoveFileLazy is the same as MoveFile, except that it avoids copying
// the file when the destination already has the same contents.
func MoveFileLazy(src, dst string) error {
	same, err := SameContents(src, dst)
	if err != nil {
		return err
	}
	if same {
		return os.Remove(src)
	}
	return MoveFile(src, dst)
}

// CopyFileLazy is the same as CopyFile, except that it avoids copying
// the file when the destination already has the same contents.
func CopyFileLazy(src, dst string) error {
	same, err := SameContents(src, dst)
	if err != nil {
		return err
	}
	if same {
		return nil
	}
	return CopyFile(src, dst)
}

// SameContents returns same = true if src and dst both exist and have the
// same file contents. Whether the file data is at the same place on
// disk is a different question, which is not answered.
//
// If either file is a directory, FileTypeError is returned.
func SameContents(src, dst string) (same bool, err error) {
	same, err = SameFile(src, dst)
	if err != nil {
		return same, err
	}

	// TODO: I could make this more efficient, based on the file size.
	ssum, err := sumFile(src)
	if err != nil {
		return false, err
	}
	dsum, err := sumFile(dst)
	if err != nil {
		return false, err
	}
	return bytes.Compare(ssum, dsum) == 0, nil
}

// sumFile creates an md5sum of a file
func sumFile(path string) (sum []byte, err error) {
	h := md5.New()
	f, err := os.Open(path)
	if err != nil {
		return sum, err
	}
	io.Copy(h, f)
	return h.Sum(nil), nil
}

func SameFile(src, dst string) (same bool, err error) {
	fs, err := os.Stat(src)
	if err != nil {
		return false, err
	}
	if fs.IsDir() {
		return false, FileTypeError{src}
	}

	fd, err := os.Stat(dst)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	if fd.IsDir() {
		return false, FileTypeError{dst}
	}

	return os.SameFile(fs, fd), nil
}

// Exists returns ex = true if the given file exists, regardless whether
// it is a file or a directory. Normally you will probably want to use the more
// specific versions: FileExists and DirectoryExists.
func Exists(path string) (ex bool, err error) {
	ex, _, err = exists(path)
	return ex, err
}

// FileExists returns ex = true if the file exists and is not
// a directory, and returns err != nil if any other error occured (such as
// permission denied).
func FileExists(path string) (ex bool, err error) {
	var stat os.FileInfo

	ex, stat, err = exists(path)
	if err == nil && stat.IsDir() {
		return false, FileTypeError{path}
	}
	return ex, err
}

// DirExists returns ex = true if the file exists and is
// a directory, and returns err != nil if any other error occured (such as
// permission denied).
func DirExists(path string) (ex bool, err error) {
	var stat os.FileInfo

	ex, stat, err = exists(path)
	if err == nil && !stat.IsDir() {
		return false, FileTypeError{path}
	}
	return ex, err
}

// exists does the hard work for Exists, FileExists, and DirExists,
// returning ex = true if the file given by path exists.
func exists(path string) (ex bool, stat os.FileInfo, err error) {
	stat, err = os.Stat(path)

	ex = true
	if err != nil {
		// ex = true if file exists
		ex = !os.IsNotExist(err)
		if !ex {
			err = nil
		}
	}
	return ex, stat, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

func main() {
	fmt.Println("Hello, 世界")
	fmt.Println("The time is ", time.Now())
}

<commit_msg>random<commit_after>package main

import (
	"fmt"
	"time"
	"math/rand"
)

func main() {
	fmt.Println("Hello, 世界")
	fmt.Println("The time is ", time.Now())
	fmt.Println("My favorite number is", rand.Intn(10))
}

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/dustin/go-id3"
	"github.com/dustin/goexif/exif"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")
var uploadMeta = uploadFlags.Bool("meta", false,
	"Store meta info in userData for items")
var uploadWorkers = uploadFlags.Int("workers", 4, "Number of upload workers")
var uploadRevs = uploadFlags.Int("revs", 0,
	"Number of old revisions to keep (-1 == all)")
var uploadNoop = uploadFlags.Bool("n", false, "Dry run")
var uploadIgnore = uploadFlags.String("ignore", "",
	"Path to ignore file")
var uploadRevsSet = false

var quotingReplacer = strings.NewReplacer("%", "%25",
	"?", "%3f",
	" ", "%20",
	"#", "%23")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

func (u uploadOpType) String() string {
	switch u {
	case uploadFileOp:
		return "upload file"
	case removeFileOp:
		return "remove file"
	case removeRecurseOp:
		return "remove (recursive) file"
	}
	panic("unhandled op type")
}

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	case strings.HasSuffix(n, ".css"):
		return "text/css"
	}
	return def
}

func processMP3Meta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ifile := id3.Read(f)
	return ifile, nil
}

func processEXIFMeta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return exif.Decode(f)
}

func processMeta(src, dest string) error {
	var data interface{}
	var err error

	switch filepath.Ext(strings.ToLower(src)) {
	case ".mp3":
		data, err = processMP3Meta(src, dest)
	case ".jpg", ".jpeg":
		data, err = processEXIFMeta(src, dest)
	default:
		log.Printf("No meta info for %#v",
			filepath.Ext(strings.ToLower(src)))
	}

	if err != nil || data == nil {
		return err
	}

	b, err := json.Marshal(data)
	if err != nil {
		return err
	}

	udest, err := url.Parse(dest)
	if err != nil {
		return err
	}
	udest.Path = "/.cbfs/meta" + udest.Path

	preq, err := http.NewRequest("PUT", udest.String(), bytes.NewReader(b))
	if err != nil {
		return err
	}

	preq.Header.Set("Content-Type", "application/json")

	if *uploadVerbose {
		log.Printf("Uploading meta info to %v", udest)
	}

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}

	return nil
}

func uploadFile(src, dest, localHash string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v (%v)", src, dest, localHash)
	}
	if *uploadNoop {
		return nil
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]

	length, err := f.Seek(0, 2)
	if err != nil {
		return err
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	if uploadRevsSet {
		preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*uploadRevs))
	}

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Length", strconv.FormatInt(length, 10))
	preq.Header.Set("Content-Type", ctype)
	preq.Header.Set("X-CBFS-Hash", localHash)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		r, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("HTTP Error:  %v: %s", resp.Status, r)
	}

	if *uploadMeta {
		err = processMeta(src, dest)
		if err != nil {
			log.Printf("Error processing meta info: %v", err)
		}
	}

	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDir(baseUrl string) ([]string, error) {
	if *uploadVerbose {
		log.Printf("Removing directory: %v", baseUrl)
	}
	r := quotingReplacer
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := cbfsclient.ListOrEmpty(baseUrl)
	for err != nil {
		return []string{}, err
	}
	for fn := range listing.Files {
		if *uploadVerbose {
			log.Printf("Removing file %v/%v", baseUrl, fn)
		}
		if !*uploadNoop {
			err = rmFile(baseUrl + "/" + r.Replace(fn))
			if err != nil {
				return []string{}, err
			}
		}
	}
	children := make([]string, 0, len(listing.Dirs))
	for dn := range listing.Dirs {
		children = append(children, baseUrl+"/"+r.Replace(dn))
	}
	return children, nil
}

func uploadRmDashR(d string) error {
	if *uploadVerbose {
		log.Printf("Removing (recursively) %v", d)
	}

	children, err := uploadRmDir(d)
	if err == nil && len(children) > 0 {
		for _, child := range children {
			err = uploadRmDashR(child)
		}
	}
	return err
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				lh := localHash(req.src)
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest, lh)
				} else {
					if lh != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest, lh)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				if !*uploadNoop {
					err = rmFile(req.dest)
				}
			case removeRecurseOp:
				err = uploadRmDashR(req.dest)
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error in %v: %v... retrying",
						req.op, err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Printf("Error in %v %v: %v",
						req.op, req.src, err)
					done = true
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	dest = quotingReplacer.Replace(dest)

	retries := 3
	serverListing, err := cbfsclient.ListOrEmpty(dest)
	for err != nil && retries > 0 {
		serverListing, err = cbfsclient.ListOrEmpty(dest)
		time.Sleep(time.Second)
		retries--
	}
	if err != nil {
		return err
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v - %v",
					filepath.Join(path, c.Name()), c.Mode())
			}
		default:
			fullPath := filepath.Join(path, c.Name())
			if !isIgnored(fullPath) {
				localNames[c.Name()] = c
			} else {
				if *uploadVerbose {
					log.Printf("Ignoring %v", fullPath)
				}
			}
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		if n != "" {
			remoteNames[n] = true
		}
	}
	for n := range serverListing.Dirs {
		if n != "" {
			remoteNames[n] = true
		}
	}

	// Keeping it short
	r := quotingReplacer

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + r.Replace(n), uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + r.Replace(m), uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				if isIgnored(path) {
					if *uploadVerbose {
						log.Printf("Skipping dir %v",
							path)
					}
					return filepath.SkipDir
				}
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(u string, args []string) {
	uploadFlags.Parse(args)

	uploadFlags.Visit(func(f *flag.Flag) {
		if f.Name == "revs" {
			uploadRevsSet = true
		}
	})

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	if *uploadIgnore != "" {
		err := loadIgnorePatternsFromFile(*uploadIgnore)
		if err != nil {
			log.Fatalf("Error loading ignores: %v", err)
		}
	}

	du := relativeUrl(u, uploadFlags.Arg(1))

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *uploadWorkers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), du, ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), du,
			localHash(uploadFlags.Arg(0)))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<commit_msg>upload .json -> application/json<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/dustin/go-id3"
	"github.com/dustin/goexif/exif"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")
var uploadMeta = uploadFlags.Bool("meta", false,
	"Store meta info in userData for items")
var uploadWorkers = uploadFlags.Int("workers", 4, "Number of upload workers")
var uploadRevs = uploadFlags.Int("revs", 0,
	"Number of old revisions to keep (-1 == all)")
var uploadNoop = uploadFlags.Bool("n", false, "Dry run")
var uploadIgnore = uploadFlags.String("ignore", "",
	"Path to ignore file")
var uploadRevsSet = false

var quotingReplacer = strings.NewReplacer("%", "%25",
	"?", "%3f",
	" ", "%20",
	"#", "%23")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

func (u uploadOpType) String() string {
	switch u {
	case uploadFileOp:
		return "upload file"
	case removeFileOp:
		return "remove file"
	case removeRecurseOp:
		return "remove (recursive) file"
	}
	panic("unhandled op type")
}

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	case strings.HasSuffix(n, ".json"):
		return "application/json"
	case strings.HasSuffix(n, ".css"):
		return "text/css"
	}
	return def
}

func processMP3Meta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ifile := id3.Read(f)
	return ifile, nil
}

func processEXIFMeta(src, dest string) (interface{}, error) {
	f, err := os.Open(src)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return exif.Decode(f)
}

func processMeta(src, dest string) error {
	var data interface{}
	var err error

	switch filepath.Ext(strings.ToLower(src)) {
	case ".mp3":
		data, err = processMP3Meta(src, dest)
	case ".jpg", ".jpeg":
		data, err = processEXIFMeta(src, dest)
	default:
		log.Printf("No meta info for %#v",
			filepath.Ext(strings.ToLower(src)))
	}

	if err != nil || data == nil {
		return err
	}

	b, err := json.Marshal(data)
	if err != nil {
		return err
	}

	udest, err := url.Parse(dest)
	if err != nil {
		return err
	}
	udest.Path = "/.cbfs/meta" + udest.Path

	preq, err := http.NewRequest("PUT", udest.String(), bytes.NewReader(b))
	if err != nil {
		return err
	}

	preq.Header.Set("Content-Type", "application/json")

	if *uploadVerbose {
		log.Printf("Uploading meta info to %v", udest)
	}

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}

	return nil
}

func uploadFile(src, dest, localHash string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v (%v)", src, dest, localHash)
	}
	if *uploadNoop {
		return nil
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]

	length, err := f.Seek(0, 2)
	if err != nil {
		return err
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	if uploadRevsSet {
		preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*uploadRevs))
	}

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Length", strconv.FormatInt(length, 10))
	preq.Header.Set("Content-Type", ctype)
	preq.Header.Set("X-CBFS-Hash", localHash)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		r, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("HTTP Error:  %v: %s", resp.Status, r)
	}

	if *uploadMeta {
		err = processMeta(src, dest)
		if err != nil {
			log.Printf("Error processing meta info: %v", err)
		}
	}

	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDir(baseUrl string) ([]string, error) {
	if *uploadVerbose {
		log.Printf("Removing directory: %v", baseUrl)
	}
	r := quotingReplacer
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := cbfsclient.ListOrEmpty(baseUrl)
	for err != nil {
		return []string{}, err
	}
	for fn := range listing.Files {
		if *uploadVerbose {
			log.Printf("Removing file %v/%v", baseUrl, fn)
		}
		if !*uploadNoop {
			err = rmFile(baseUrl + "/" + r.Replace(fn))
			if err != nil {
				return []string{}, err
			}
		}
	}
	children := make([]string, 0, len(listing.Dirs))
	for dn := range listing.Dirs {
		children = append(children, baseUrl+"/"+r.Replace(dn))
	}
	return children, nil
}

func uploadRmDashR(d string) error {
	if *uploadVerbose {
		log.Printf("Removing (recursively) %v", d)
	}

	children, err := uploadRmDir(d)
	if err == nil && len(children) > 0 {
		for _, child := range children {
			err = uploadRmDashR(child)
		}
	}
	return err
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				lh := localHash(req.src)
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest, lh)
				} else {
					if lh != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest, lh)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				if !*uploadNoop {
					err = rmFile(req.dest)
				}
			case removeRecurseOp:
				err = uploadRmDashR(req.dest)
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error in %v: %v... retrying",
						req.op, err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Printf("Error in %v %v: %v",
						req.op, req.src, err)
					done = true
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	dest = quotingReplacer.Replace(dest)

	retries := 3
	serverListing, err := cbfsclient.ListOrEmpty(dest)
	for err != nil && retries > 0 {
		serverListing, err = cbfsclient.ListOrEmpty(dest)
		time.Sleep(time.Second)
		retries--
	}
	if err != nil {
		return err
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v - %v",
					filepath.Join(path, c.Name()), c.Mode())
			}
		default:
			fullPath := filepath.Join(path, c.Name())
			if !isIgnored(fullPath) {
				localNames[c.Name()] = c
			} else {
				if *uploadVerbose {
					log.Printf("Ignoring %v", fullPath)
				}
			}
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		if n != "" {
			remoteNames[n] = true
		}
	}
	for n := range serverListing.Dirs {
		if n != "" {
			remoteNames[n] = true
		}
	}

	// Keeping it short
	r := quotingReplacer

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + r.Replace(n), uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + r.Replace(m), uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + r.Replace(m), removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				if isIgnored(path) {
					if *uploadVerbose {
						log.Printf("Skipping dir %v",
							path)
					}
					return filepath.SkipDir
				}
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(u string, args []string) {
	uploadFlags.Parse(args)

	uploadFlags.Visit(func(f *flag.Flag) {
		if f.Name == "revs" {
			uploadRevsSet = true
		}
	})

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	if *uploadIgnore != "" {
		err := loadIgnorePatternsFromFile(*uploadIgnore)
		if err != nil {
			log.Fatalf("Error loading ignores: %v", err)
		}
	}

	du := relativeUrl(u, uploadFlags.Arg(1))

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *uploadWorkers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), du, ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), du,
			localHash(uploadFlags.Arg(0)))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "bytes"
    "fmt"
    "io"
    "io/ioutil"
    "mime/multipart"
    "net/http"
    "os"
)

func postFile(filename string, targetUrl string) error {
    bodyBuf := &bytes.Buffer{}
    bodyWriter := multipart.NewWriter(bodyBuf)

    // this step is very important
    fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename)
    if err != nil {
        fmt.Println("error writing to buffer")
        return err
    }

    // open file handle
    fh, err := os.Open(filename)
    if err != nil {
        fmt.Println("error opening file")
        return err
    }

    //iocopy
    _, err = io.Copy(fileWriter, fh)
    if err != nil {
        return err
    }

    contentType := bodyWriter.FormDataContentType()
    bodyWriter.Close()

    resp, err := http.Post(targetUrl, contentType, bodyBuf)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
    resp_body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return err
    }
    fmt.Println(resp.Status)
    fmt.Println(string(resp_body))
    return nil
}

// sample usage
func main() {
    target_url := "http://localhost:6543/api/upload"
    filename := "./a.webm"
    postFile(filename, target_url)
}
<commit_msg>chore: update api path and test file's name<commit_after>package main

import (
    "bytes"
    "fmt"
    "io"
    "io/ioutil"
    "mime/multipart"
    "net/http"
    "os"
)

func postFile(filename string, targetUrl string) error {
    bodyBuf := &bytes.Buffer{}
    bodyWriter := multipart.NewWriter(bodyBuf)

    // this step is very important
    fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename)
    if err != nil {
        fmt.Println("error writing to buffer")
        return err
    }

    // open file handle
    fh, err := os.Open(filename)
    if err != nil {
        fmt.Println("error opening file")
        return err
    }

    //iocopy
    _, err = io.Copy(fileWriter, fh)
    if err != nil {
        return err
    }

    contentType := bodyWriter.FormDataContentType()
    bodyWriter.Close()

    resp, err := http.Post(targetUrl, contentType, bodyBuf)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
    resp_body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return err
    }
    fmt.Println(resp.Status)
    fmt.Println(string(resp_body))
    return nil
}

// sample usage
func main() {
    target_url := "http://localhost:6540/api/upload"
    filename := "a.webm"
    postFile(filename, target_url)
}
<|endoftext|>


<commit_before>package arn

import (
	"sort"
	"time"
)

const (
	currentlyAiringBonus      = 5.0
	longSummaryBonus          = 0.1
	popularityThreshold       = 5
	popularityPenalty         = 8.0
	watchingPopularityWeight  = 0.3
	completedPopularityWeight = 0.3
	plannedPopularityWeight   = 0.2
	droppedPopularityWeight   = -0.2
	agePenalty                = 11.0
	ageThreshold              = 6 * 30 * 24 * time.Hour
)

// SortAnimeByPopularity sorts the given slice of anime by popularity.
func SortAnimeByPopularity(animes []*Anime) {
	sort.Slice(animes, func(i, j int) bool {
		aPopularity := animes[i].Popularity.Total()
		bPopularity := animes[j].Popularity.Total()

		if aPopularity == bPopularity {
			return animes[i].Title.Canonical < animes[j].Title.Canonical
		}

		return aPopularity > bPopularity
	})
}

// SortAnimeByQuality sorts the given slice of anime by quality.
func SortAnimeByQuality(animes []*Anime) {
	SortAnimeByQualityDetailed(animes, "")
}

// SortAnimeByQualityDetailed sorts the given slice of anime by quality.
func SortAnimeByQualityDetailed(animes []*Anime, filterStatus string) {
	sort.Slice(animes, func(i, j int) bool {
		a := animes[i]
		b := animes[j]

		scoreA := a.Rating.Overall
		scoreB := b.Rating.Overall

		if a.Status == "current" {
			scoreA += currentlyAiringBonus
		}

		if b.Status == "current" {
			scoreB += currentlyAiringBonus
		}

		if a.Popularity.Total() < popularityThreshold {
			scoreA -= popularityPenalty
		}

		if b.Popularity.Total() < popularityThreshold {
			scoreB -= popularityPenalty
		}

		if len(a.Summary) >= 140 {
			scoreA += longSummaryBonus
		}

		if len(b.Summary) >= 140 {
			scoreB += longSummaryBonus
		}

		// If we show currently running shows, rank shows that started a long time ago a bit lower
		if filterStatus == "current" {
			if a.StartDate != "" && time.Since(a.StartDateTime()) > ageThreshold {
				scoreA -= agePenalty
			}

			if b.StartDate != "" && time.Since(b.StartDateTime()) > ageThreshold {
				scoreB -= agePenalty
			}
		}

		scoreA += float64(a.Popularity.Watching) * watchingPopularityWeight
		scoreB += float64(b.Popularity.Watching) * watchingPopularityWeight

		scoreA += float64(a.Popularity.Planned) * plannedPopularityWeight
		scoreB += float64(b.Popularity.Planned) * plannedPopularityWeight

		scoreA += float64(a.Popularity.Completed) * completedPopularityWeight
		scoreB += float64(b.Popularity.Completed) * completedPopularityWeight

		scoreA += float64(a.Popularity.Dropped) * droppedPopularityWeight
		scoreB += float64(b.Popularity.Dropped) * droppedPopularityWeight

		return scoreA > scoreB
	})
}
<commit_msg>Updated weightings<commit_after>package arn

import (
	"sort"
	"time"
)

const (
	currentlyAiringBonus      = 5.0
	longSummaryBonus          = 0.1
	popularityThreshold       = 5
	popularityPenalty         = 8.0
	watchingPopularityWeight  = 0.25
	completedPopularityWeight = watchingPopularityWeight
	plannedPopularityWeight   = watchingPopularityWeight * 0.666
	droppedPopularityWeight   = -plannedPopularityWeight
	agePenalty                = 11.0
	ageThreshold              = 6 * 30 * 24 * time.Hour
)

// SortAnimeByPopularity sorts the given slice of anime by popularity.
func SortAnimeByPopularity(animes []*Anime) {
	sort.Slice(animes, func(i, j int) bool {
		aPopularity := animes[i].Popularity.Total()
		bPopularity := animes[j].Popularity.Total()

		if aPopularity == bPopularity {
			return animes[i].Title.Canonical < animes[j].Title.Canonical
		}

		return aPopularity > bPopularity
	})
}

// SortAnimeByQuality sorts the given slice of anime by quality.
func SortAnimeByQuality(animes []*Anime) {
	SortAnimeByQualityDetailed(animes, "")
}

// SortAnimeByQualityDetailed sorts the given slice of anime by quality.
func SortAnimeByQualityDetailed(animes []*Anime, filterStatus string) {
	sort.Slice(animes, func(i, j int) bool {
		a := animes[i]
		b := animes[j]

		scoreA := a.Rating.Overall
		scoreB := b.Rating.Overall

		if a.Status == "current" {
			scoreA += currentlyAiringBonus
		}

		if b.Status == "current" {
			scoreB += currentlyAiringBonus
		}

		if a.Popularity.Total() < popularityThreshold {
			scoreA -= popularityPenalty
		}

		if b.Popularity.Total() < popularityThreshold {
			scoreB -= popularityPenalty
		}

		if len(a.Summary) >= 140 {
			scoreA += longSummaryBonus
		}

		if len(b.Summary) >= 140 {
			scoreB += longSummaryBonus
		}

		// If we show currently running shows, rank shows that started a long time ago a bit lower
		if filterStatus == "current" {
			if a.StartDate != "" && time.Since(a.StartDateTime()) > ageThreshold {
				scoreA -= agePenalty
			}

			if b.StartDate != "" && time.Since(b.StartDateTime()) > ageThreshold {
				scoreB -= agePenalty
			}
		}

		scoreA += float64(a.Popularity.Watching) * watchingPopularityWeight
		scoreB += float64(b.Popularity.Watching) * watchingPopularityWeight

		scoreA += float64(a.Popularity.Planned) * plannedPopularityWeight
		scoreB += float64(b.Popularity.Planned) * plannedPopularityWeight

		scoreA += float64(a.Popularity.Completed) * completedPopularityWeight
		scoreB += float64(b.Popularity.Completed) * completedPopularityWeight

		scoreA += float64(a.Popularity.Dropped) * droppedPopularityWeight
		scoreB += float64(b.Popularity.Dropped) * droppedPopularityWeight

		return scoreA > scoreB
	})
}
<|endoftext|>


<commit_before>package changelist

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/Sirupsen/logrus"
)

type fileChangelist struct {
	dir string
}

func NewFileChangelist(dir string) (*fileChangelist, error) {
	logrus.Debug("Making dir path: ", dir)
	err := os.MkdirAll(dir, 0700)
	if err != nil {
		return nil, err
	}
	return &fileChangelist{dir: dir}, nil
}

func (cl fileChangelist) List() []Change {
	changes := make([]Change, 0)
	dir, err := os.Open(cl.dir)
	if err != nil {
		return changes
	}
	fileInfos, err := dir.Readdir(0)
	if err != nil {
		return changes
	}
	sort.Sort(fileChanges(fileInfos))
	for _, f := range fileInfos {
		if f.IsDir() {
			continue
		}
		raw, err := ioutil.ReadFile(path.Join(cl.dir, f.Name()))
		if err != nil {
			// TODO(david): How should we handle this?
			fmt.Println(err.Error())
			continue
		}
		c := &tufChange{}
		err = json.Unmarshal(raw, c)
		if err != nil {
			// TODO(david): How should we handle this?
			fmt.Println(err.Error())
			continue
		}
		changes = append(changes, c)
	}
	return changes
}

func (cl fileChangelist) Add(c Change) error {
	cJson, err := json.Marshal(c)
	if err != nil {
		return err
	}
	filename := fmt.Sprintf("%020d_%s.change", time.Now().UnixNano(), uuid.New())
	return ioutil.WriteFile(path.Join(cl.dir, filename), cJson, 0644)
}

func (cl fileChangelist) Clear(archive string) error {
	dir, err := os.Open(cl.dir)
	if err != nil {
		return err
	}
	files, err := dir.Readdir(0)
	if err != nil {
		return err
	}
	for _, f := range files {
		os.Remove(path.Join(cl.dir, f.Name()))
	}
	return nil
}

func (cl fileChangelist) Close() error {
	// Nothing to do here
	return nil
}

type fileChanges []os.FileInfo

func (cs fileChanges) Len() int {
	return len(cs)
}

func (cs fileChanges) Less(i, j int) bool {
	return cs[i].Name() < cs[j].Name()
}

func (cs fileChanges) Swap(i, j int) {
	tmp := cs[i]
	cs[i] = cs[j]
	cs[j] = tmp
}
<commit_msg>Add missing Close calls to address file descriptor leak<commit_after>package changelist

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/Sirupsen/logrus"
)

type fileChangelist struct {
	dir string
}

func NewFileChangelist(dir string) (*fileChangelist, error) {
	logrus.Debug("Making dir path: ", dir)
	err := os.MkdirAll(dir, 0700)
	if err != nil {
		return nil, err
	}
	return &fileChangelist{dir: dir}, nil
}

func (cl fileChangelist) List() []Change {
	changes := make([]Change, 0)
	dir, err := os.Open(cl.dir)
	if err != nil {
		return changes
	}
	defer dir.Close()
	fileInfos, err := dir.Readdir(0)
	if err != nil {
		return changes
	}
	sort.Sort(fileChanges(fileInfos))
	for _, f := range fileInfos {
		if f.IsDir() {
			continue
		}
		raw, err := ioutil.ReadFile(path.Join(cl.dir, f.Name()))
		if err != nil {
			// TODO(david): How should we handle this?
			fmt.Println(err.Error())
			continue
		}
		c := &tufChange{}
		err = json.Unmarshal(raw, c)
		if err != nil {
			// TODO(david): How should we handle this?
			fmt.Println(err.Error())
			continue
		}
		changes = append(changes, c)
	}
	return changes
}

func (cl fileChangelist) Add(c Change) error {
	cJson, err := json.Marshal(c)
	if err != nil {
		return err
	}
	filename := fmt.Sprintf("%020d_%s.change", time.Now().UnixNano(), uuid.New())
	return ioutil.WriteFile(path.Join(cl.dir, filename), cJson, 0644)
}

func (cl fileChangelist) Clear(archive string) error {
	dir, err := os.Open(cl.dir)
	if err != nil {
		return err
	}
	defer dir.Close()
	files, err := dir.Readdir(0)
	if err != nil {
		return err
	}
	for _, f := range files {
		os.Remove(path.Join(cl.dir, f.Name()))
	}
	return nil
}

func (cl fileChangelist) Close() error {
	// Nothing to do here
	return nil
}

type fileChanges []os.FileInfo

func (cs fileChanges) Len() int {
	return len(cs)
}

func (cs fileChanges) Less(i, j int) bool {
	return cs[i].Name() < cs[j].Name()
}

func (cs fileChanges) Swap(i, j int) {
	tmp := cs[i]
	cs[i] = cs[j]
	cs[j] = tmp
}
<|endoftext|>


<commit_before>package libstns

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/STNS/libnss_stns/cache"
)

type Request struct {
	ApiPath      string
	Config       *Config
	ResourceType string
}

func NewRequest(config *Config, paths ...string) (*Request, error) {
	if len(paths) == 3 && strings.Contains(paths[2], ".") {
		paths[2] = urlencode(paths[2])
	}

	r := Request{
		ApiPath:      path.Clean(strings.Join(paths, "/")),
		Config:       config,
		ResourceType: paths[0],
	}
	return &r, nil
}

// only use wrapper command
func (r *Request) GetRawData() ([]byte, error) {
	var b []byte
	var e error

	if len(r.Config.ApiEndPoint) == 0 {
		return nil, errors.New("endpoint not defined")
	}

	retry := 3
	if r.Config.RequestRetry != 0 {
		retry = r.Config.RequestRetry
	}

	for i := 0; i < retry; i++ {
		b, e = r.request()
		if e == nil {
			break
		}
	}
	return b, e
}

func (r *Request) request() ([]byte, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	rch := make(chan *ResponseFormat, len(r.Config.ApiEndPoint))
	ech := make(chan error, len(r.Config.ApiEndPoint))
	for _, e := range r.Config.ApiEndPoint {
		go func(endPoint string) {
			if cache.IsLockEndPoint(endPoint) {
				ech <- fmt.Errorf("endpoint %s is locked", endPoint)
				return
			}

			u := strings.TrimRight(endPoint, "/") + "/" + strings.TrimLeft(r.ApiPath, "/")
			req, err := http.NewRequest("GET", u, nil)
			if err != nil {
				ech <- err
				return
			}

			for k, v := range r.Config.RequestHeader {
				req.Header.Add(k, v)
			}

			if r.Config.User != "" && r.Config.Password != "" {
				req.SetBasicAuth(r.Config.User, r.Config.Password)
			}

			r.httpDo(
				ctx,
				req,
				func(res *http.Response, err error) {
					if err != nil {
						if _, ok := err.(*url.Error); ok && len(r.Config.ApiEndPoint) != 1 {
							cache.LockEndPoint(endPoint)
						}
						ech <- err
						return
					}

					defer res.Body.Close()
					body, err := ioutil.ReadAll(res.Body)

					switch res.StatusCode {
					case http.StatusOK:
						v2 := regexp.MustCompile(`/v2[/]?$`)
						v3 := regexp.MustCompile(`/v3[/]?$`)
						switch {
						// version1
						case !v2.MatchString(endPoint) && !v3.MatchString(endPoint):
							buffer, err := convertV1toV3Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						// version2
						case v2.MatchString(endPoint):
							buffer, err := convertV2toV3Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						default:
							buffer, err := convertV3Format(body, r.ApiPath, r.Config)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						}
					case http.StatusNotFound:
						ids := map[string]int{}
						for _, t := range []string{"Min", "Max"} {
							id := res.Header.Get(fmt.Sprintf("Stns-%s-Id", t))
							if id != "" {
								i, err := strconv.Atoi(id)
								if err != nil {
									ech <- err
									return
								}
								ids[t] = i
							}
						}

						if len(ids) > 0 {
							ech <- fmt.Errorf("resource not found min_id: %v max_id %v url: %s", ids["Min"], ids["Max"], u)
						} else {
							ech <- fmt.Errorf("resource not found url: %s", u)
						}
						return
					case http.StatusUnauthorized:
						ech <- fmt.Errorf("authenticate error: %s", u)
						return
					default:
						ech <- fmt.Errorf("error: %s", u)
						return
					}
				},
			)
		}(e)
	}

	var cnt int
	for {
		select {
		case r := <-rch:
			j, err := json.Marshal(r)
			if err != nil {
				return nil, err
			}
			return j, nil
		case e := <-ech:
			cnt++
			if cnt == len(r.Config.ApiEndPoint) {
				return nil, e
			}
		}
	}

}
func (r *Request) httpDo(
	ctx context.Context,
	req *http.Request,
	f func(*http.Response, error),
) {
	tc := r.TlsConfig()
	tr := &http.Transport{
		TLSClientConfig: tc,
		Dial: (&net.Dialer{
			Timeout:   time.Duration(r.Config.RequestTimeOut) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	tr.Proxy = http.ProxyFromEnvironment
	if r.Config.HttpProxy != "" {
		proxyUrl, err := url.Parse(r.Config.HttpProxy)
		if err == nil {
			tr.Proxy = http.ProxyURL(proxyUrl)
		}
	}

	client := &http.Client{Transport: tr}

	go func() { f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		return
	}
}

func (r *Request) TlsConfig() *tls.Config {
	tc := &tls.Config{InsecureSkipVerify: !r.Config.SslVerify}

	if r.TlsKeysExists() {
		cert, err := tls.LoadX509KeyPair(r.Config.TlsCert, r.Config.TlsKey)
		if err != nil {
			log.Println(err)
			goto ret
		}

		if _, err := os.Stat(r.Config.TlsCa); err == nil {
			// Load CA cert
			caCert, err := ioutil.ReadFile(r.Config.TlsCa)
			if err != nil {
				log.Println(err)
				goto ret
			}
			caPool := x509.NewCertPool()
			caPool.AppendCertsFromPEM(caCert)

			tc.Certificates = []tls.Certificate{cert}
			tc.RootCAs = caPool

			tc.BuildNameToCertificate()
		}

	}
ret:
	return tc
}

func (r *Request) TlsKeysExists() bool {
	if r.Config.TlsCert != "" && r.Config.TlsKey != "" {
		for _, v := range []string{r.Config.TlsCert, r.Config.TlsKey} {
			if _, err := os.Stat(v); err != nil {
				log.Println(err)
				return false
			}
		}
		return true
	}
	return false
}

func (r *Request) GetByWrapperCmd() (*ResponseFormat, error) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(r.Config.WrapperCommand, r.ApiPath)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()

	if err != nil {
		return nil, err
	}

	if len(stderr.Bytes()) > 0 {
		reg := regexp.MustCompile(`resource not found min_id: ([\d]+) max_id ([\d]+) url: .*`)
		if result := reg.FindStringSubmatch(stderr.String()); result != nil {
			for index, t := range []string{"min", "max"} {
				i, err := strconv.Atoi(string(result[index+1]))
				if err != nil {
					return nil, fmt.Errorf("command error:%s", err)
				}
				cache.WriteID(r.ResourceType, t, i)
			}
		}
		return nil, fmt.Errorf("command error:%s", stderr.String())
	}

	var res *ResponseFormat
	if len(stdout.Bytes()) > 0 {
		err = json.Unmarshal(stdout.Bytes(), &res)
		if err != nil {
			return nil, err
		}
	}
	return res, nil
}

func urlencode(s string) (result string) {
	for _, c := range s {
		if c <= 0x7f { // single byte
			result += fmt.Sprintf("%%%X", c)
		} else if c > 0x1fffff { // quaternary byte
			result += fmt.Sprintf("%%%X%%%X%%%X%%%X",
				0xf0+((c&0x1c0000)>>18),
				0x80+((c&0x3f000)>>12),
				0x80+((c&0xfc0)>>6),
				0x80+(c&0x3f),
			)
		} else if c > 0x7ff { // triple byte
			result += fmt.Sprintf("%%%X%%%X%%%X",
				0xe0+((c&0xf000)>>12),
				0x80+((c&0xfc0)>>6),
				0x80+(c&0x3f),
			)
		} else { // double byte
			result += fmt.Sprintf("%%%X%%%X",
				0xc0+((c&0x7c0)>>6),
				0x80+(c&0x3f),
			)
		}
	}

	return result
}
<commit_msg>check error<commit_after>package libstns

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/STNS/libnss_stns/cache"
)

type Request struct {
	ApiPath      string
	Config       *Config
	ResourceType string
}

func NewRequest(config *Config, paths ...string) (*Request, error) {
	if len(paths) == 3 && strings.Contains(paths[2], ".") {
		paths[2] = urlencode(paths[2])
	}

	r := Request{
		ApiPath:      path.Clean(strings.Join(paths, "/")),
		Config:       config,
		ResourceType: paths[0],
	}
	return &r, nil
}

// only use wrapper command
func (r *Request) GetRawData() ([]byte, error) {
	var b []byte
	var e error

	if len(r.Config.ApiEndPoint) == 0 {
		return nil, errors.New("endpoint not defined")
	}

	retry := 3
	if r.Config.RequestRetry != 0 {
		retry = r.Config.RequestRetry
	}

	for i := 0; i < retry; i++ {
		b, e = r.request()
		if e == nil {
			break
		}

		if strings.Index(e.Error(), "resource not found") >= 0 {
			return nil, e
		}
	}
	return b, e
}

func (r *Request) request() ([]byte, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	rch := make(chan *ResponseFormat, len(r.Config.ApiEndPoint))
	ech := make(chan error, len(r.Config.ApiEndPoint))
	for _, e := range r.Config.ApiEndPoint {
		go func(endPoint string) {
			if cache.IsLockEndPoint(endPoint) {
				ech <- fmt.Errorf("endpoint %s is locked", endPoint)
				return
			}

			u := strings.TrimRight(endPoint, "/") + "/" + strings.TrimLeft(r.ApiPath, "/")
			req, err := http.NewRequest("GET", u, nil)
			if err != nil {
				ech <- err
				return
			}

			for k, v := range r.Config.RequestHeader {
				req.Header.Add(k, v)
			}

			if r.Config.User != "" && r.Config.Password != "" {
				req.SetBasicAuth(r.Config.User, r.Config.Password)
			}

			r.httpDo(
				ctx,
				req,
				func(res *http.Response, err error) {
					if err != nil {
						if _, ok := err.(*url.Error); ok && len(r.Config.ApiEndPoint) != 1 {
							cache.LockEndPoint(endPoint)
						}
						ech <- err
						return
					}

					defer res.Body.Close()
					body, err := ioutil.ReadAll(res.Body)

					switch res.StatusCode {
					case http.StatusOK:
						v2 := regexp.MustCompile(`/v2[/]?$`)
						v3 := regexp.MustCompile(`/v3[/]?$`)
						switch {
						// version1
						case !v2.MatchString(endPoint) && !v3.MatchString(endPoint):
							buffer, err := convertV1toV3Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						// version2
						case v2.MatchString(endPoint):
							buffer, err := convertV2toV3Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						default:
							buffer, err := convertV3Format(body, r.ApiPath, r.Config)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						}
					case http.StatusNotFound:
						ids := map[string]int{}
						for _, t := range []string{"Min", "Max"} {
							id := res.Header.Get(fmt.Sprintf("Stns-%s-Id", t))
							if id != "" {
								i, err := strconv.Atoi(id)
								if err != nil {
									ech <- err
									return
								}
								ids[t] = i
							}
						}

						if len(ids) > 0 {
							ech <- fmt.Errorf("resource not found min_id: %v max_id %v url: %s", ids["Min"], ids["Max"], u)
						} else {
							ech <- fmt.Errorf("resource not found url: %s", u)
						}
						return
					case http.StatusUnauthorized:
						ech <- fmt.Errorf("authenticate error: %s", u)
						return
					default:
						ech <- fmt.Errorf("error: %s", u)
						return
					}
				},
			)
		}(e)
	}

	var cnt int
	for {
		select {
		case r := <-rch:
			j, err := json.Marshal(r)
			if err != nil {
				return nil, err
			}
			return j, nil
		case e := <-ech:
			cnt++
			if cnt == len(r.Config.ApiEndPoint) {
				return nil, e
			}
		}
	}

}
func (r *Request) httpDo(
	ctx context.Context,
	req *http.Request,
	f func(*http.Response, error),
) {
	tc := r.TlsConfig()
	tr := &http.Transport{
		TLSClientConfig: tc,
		Dial: (&net.Dialer{
			Timeout:   time.Duration(r.Config.RequestTimeOut) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	tr.Proxy = http.ProxyFromEnvironment
	if r.Config.HttpProxy != "" {
		proxyUrl, err := url.Parse(r.Config.HttpProxy)
		if err == nil {
			tr.Proxy = http.ProxyURL(proxyUrl)
		}
	}

	client := &http.Client{Transport: tr}

	go func() { f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		return
	}
}

func (r *Request) TlsConfig() *tls.Config {
	tc := &tls.Config{InsecureSkipVerify: !r.Config.SslVerify}

	if r.TlsKeysExists() {
		cert, err := tls.LoadX509KeyPair(r.Config.TlsCert, r.Config.TlsKey)
		if err != nil {
			log.Println(err)
			goto ret
		}

		if _, err := os.Stat(r.Config.TlsCa); err == nil {
			// Load CA cert
			caCert, err := ioutil.ReadFile(r.Config.TlsCa)
			if err != nil {
				log.Println(err)
				goto ret
			}
			caPool := x509.NewCertPool()
			caPool.AppendCertsFromPEM(caCert)

			tc.Certificates = []tls.Certificate{cert}
			tc.RootCAs = caPool

			tc.BuildNameToCertificate()
		}

	}
ret:
	return tc
}

func (r *Request) TlsKeysExists() bool {
	if r.Config.TlsCert != "" && r.Config.TlsKey != "" {
		for _, v := range []string{r.Config.TlsCert, r.Config.TlsKey} {
			if _, err := os.Stat(v); err != nil {
				log.Println(err)
				return false
			}
		}
		return true
	}
	return false
}

func (r *Request) GetByWrapperCmd() (*ResponseFormat, error) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(r.Config.WrapperCommand, r.ApiPath)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()

	if err != nil {
		return nil, err
	}

	if len(stderr.Bytes()) > 0 {
		reg := regexp.MustCompile(`resource not found min_id: ([\d]+) max_id ([\d]+) url: .*`)
		if result := reg.FindStringSubmatch(stderr.String()); result != nil {
			for index, t := range []string{"min", "max"} {
				i, err := strconv.Atoi(string(result[index+1]))
				if err != nil {
					return nil, fmt.Errorf("command error:%s", err)
				}
				cache.WriteID(r.ResourceType, t, i)
			}
		}
		return nil, fmt.Errorf("command error:%s", stderr.String())
	}

	var res *ResponseFormat
	if len(stdout.Bytes()) > 0 {
		err = json.Unmarshal(stdout.Bytes(), &res)
		if err != nil {
			return nil, err
		}
	}
	return res, nil
}

func urlencode(s string) (result string) {
	for _, c := range s {
		if c <= 0x7f { // single byte
			result += fmt.Sprintf("%%%X", c)
		} else if c > 0x1fffff { // quaternary byte
			result += fmt.Sprintf("%%%X%%%X%%%X%%%X",
				0xf0+((c&0x1c0000)>>18),
				0x80+((c&0x3f000)>>12),
				0x80+((c&0xfc0)>>6),
				0x80+(c&0x3f),
			)
		} else if c > 0x7ff { // triple byte
			result += fmt.Sprintf("%%%X%%%X%%%X",
				0xe0+((c&0xf000)>>12),
				0x80+((c&0xfc0)>>6),
				0x80+(c&0x3f),
			)
		} else { // double byte
			result += fmt.Sprintf("%%%X%%%X",
				0xc0+((c&0x7c0)>>6),
				0x80+(c&0x3f),
			)
		}
	}

	return result
}
<|endoftext|>


<commit_before>package web_test

import (
	"fmt"
	"time"

	yaml "gopkg.in/yaml.v2"

	"github.com/concourse/atc"
	"github.com/concourse/testflight/gitserver"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/matchers"
)

var _ = Describe("BuildsView", func() {
	var build atc.Build

	Context("with a job in the configuration", func() {
		var originGitServer *gitserver.Server

		BeforeEach(func() {
			originGitServer = gitserver.Start(client)
			originGitServer.CommitResource()

			config := atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "some-job",
						Plan: atc.PlanSequence{
							{
								Get: "some-input-resource",
							},
							{
								Task: "some-task",
								TaskConfig: &atc.TaskConfig{
									Platform: "linux",
									ImageResource: &atc.ImageResource{
										Type:   "docker-image",
										Source: atc.Source{"repository": "busybox"},
									},
									Inputs: []atc.TaskInputConfig{
										{Name: "some-input-resource"},
									},
									Outputs: []atc.TaskOutputConfig{
										{Name: "some-output-src"},
									},
									Run: atc.TaskRunConfig{
										Path: "cp",
										Args: []string{"-r", "some-input-resource/.", "some-output-src"},
									},
								},
							},
							{
								Put:    "some-output-resource",
								Params: atc.Params{"repository": "some-output-src"},
							},
						},
					},
				},
				Resources: []atc.ResourceConfig{
					{
						Name: "some-input-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
					{
						Name: "some-output-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
				},
			}

			byteConfig, err := yaml.Marshal(config)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, "0", byteConfig)
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			build, err = team.CreateJobBuild(pipelineName, "some-job")
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			originGitServer.Stop()
		})

		It("can view resource information of a job build", func() {
			url := atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job", teamName, pipelineName))

			Expect(page.Navigate(url)).To(Succeed())
			Eventually(page.Find(".build-header.succeeded")).Should(BeFound())

			Eventually(page.All(".builds-list li")).Should(HaveCount(1))

			Expect(page.Find(".builds-list li:first-child a")).To(HaveText("#1"))
			Eventually(page.Find(".builds-list li:first-child a.succeeded"), 10*time.Second).Should(BeFound())

			Eventually(page.Find(".builds-list li:first-child .inputs .resource-name"), 10*time.Second).Should(BeFound())
			Expect(page.Find(".builds-list li:first-child .inputs .resource-name")).To(HaveText("some-input-resource"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-value")).To(MatchText("[0-9a-f]{40}"))

			Expect(page.Find(".builds-list li:first-child .outputs .resource-name")).To(HaveText("some-output-resource"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-value")).To(MatchText("[0-9a-z]{40}"))

			// button should not have the boolean attribute "disabled" set. agouti currently returns
			// an empty string in that case.
			Expect(page.Find("button.build-action")).ToNot(BeNil())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", ""))
		})

		Describe("paused pipeline", func() {
			BeforeEach(func() {
				_, err := team.PausePipeline(pipelineName)
				Expect(err).NotTo(HaveOccurred())
			})

			It("displays a blue header", func() {
				Expect(page.Navigate(atcRoute(build.URL))).To(Succeed())

				Expect(page.Navigate(atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job/builds/%s", teamName, pipelineName, build.Name)))).To(Succeed())

				// top bar should show the pipeline is paused
				Eventually(page.Find(".top-bar.test.paused"), 10*time.Second).Should(BeFound())
			})
		})
	})

	Context("when manual triggering of the job is disabled", func() {
		var manualTriggerDisabledBuild atc.Build

		BeforeEach(func() {
			config := atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "job-manual-trigger-disabled",
					},
				},
			}

			byteConfig, err := yaml.Marshal(config)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, "0", byteConfig)
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			manualTriggerDisabledBuild, err = team.CreateJobBuild(pipelineName, "job-manual-trigger-disabled")
			Expect(err).NotTo(HaveOccurred())

			_, _, pipelineVersion, _, err := team.PipelineConfig(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			config = atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name:                 "job-manual-trigger-disabled",
						DisableManualTrigger: true,
					},
				},
			}

			byteConfig, err = yaml.Marshal(config)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, pipelineVersion, byteConfig)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should have a disabled button in the build details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))
			Eventually(page.Find("button.build-action"), 10*time.Second).Should(HaveAttribute("disabled", "true"))
		})

		It("should have a disabled button in the job details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))

			Eventually(page.Find("h1 a"), 10*time.Second).Should(BeFound())
			Expect(page.Find("h1 a").Click()).To(Succeed())
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled",
				teamName,
				pipelineName,
			))))

			Eventually(page.Find("button.build-action")).Should(BeFound())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", "true"))
		})
	})
})
<commit_msg>button -> span<commit_after>package web_test

import (
	"fmt"
	"time"

	yaml "gopkg.in/yaml.v2"

	"github.com/concourse/atc"
	"github.com/concourse/testflight/gitserver"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/matchers"
)

var _ = Describe("BuildsView", func() {
	var build atc.Build

	Context("with a job in the configuration", func() {
		var originGitServer *gitserver.Server

		BeforeEach(func() {
			originGitServer = gitserver.Start(client)
			originGitServer.CommitResource()

			config := atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "some-job",
						Plan: atc.PlanSequence{
							{
								Get: "some-input-resource",
							},
							{
								Task: "some-task",
								TaskConfig: &atc.TaskConfig{
									Platform: "linux",
									ImageResource: &atc.ImageResource{
										Type:   "docker-image",
										Source: atc.Source{"repository": "busybox"},
									},
									Inputs: []atc.TaskInputConfig{
										{Name: "some-input-resource"},
									},
									Outputs: []atc.TaskOutputConfig{
										{Name: "some-output-src"},
									},
									Run: atc.TaskRunConfig{
										Path: "cp",
										Args: []string{"-r", "some-input-resource/.", "some-output-src"},
									},
								},
							},
							{
								Put:    "some-output-resource",
								Params: atc.Params{"repository": "some-output-src"},
							},
						},
					},
				},
				Resources: []atc.ResourceConfig{
					{
						Name: "some-input-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
					{
						Name: "some-output-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
				},
			}

			byteConfig, err := yaml.Marshal(config)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, "0", byteConfig)
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			build, err = team.CreateJobBuild(pipelineName, "some-job")
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			originGitServer.Stop()
		})

		It("can view resource information of a job build", func() {
			url := atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job", teamName, pipelineName))

			Expect(page.Navigate(url)).To(Succeed())
			Eventually(page.Find(".build-header.succeeded")).Should(BeFound())

			Eventually(page.All(".builds-list li")).Should(HaveCount(1))

			Expect(page.Find(".builds-list li:first-child a")).To(HaveText("#1"))
			Eventually(page.Find(".builds-list li:first-child a.succeeded"), 10*time.Second).Should(BeFound())

			Eventually(page.Find(".builds-list li:first-child .inputs .resource-name"), 10*time.Second).Should(BeFound())
			Expect(page.Find(".builds-list li:first-child .inputs .resource-name")).To(HaveText("some-input-resource"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-value")).To(MatchText("[0-9a-f]{40}"))

			Expect(page.Find(".builds-list li:first-child .outputs .resource-name")).To(HaveText("some-output-resource"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-value")).To(MatchText("[0-9a-z]{40}"))

			// button should not have the boolean attribute "disabled" set. agouti currently returns
			// an empty string in that case.
			Expect(page.Find("button.build-action")).ToNot(BeNil())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", ""))
		})

		Describe("paused pipeline", func() {
			BeforeEach(func() {
				_, err := team.PausePipeline(pipelineName)
				Expect(err).NotTo(HaveOccurred())
			})

			It("displays a blue header", func() {
				Expect(page.Navigate(atcRoute(build.URL))).To(Succeed())

				Expect(page.Navigate(atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job/builds/%s", teamName, pipelineName, build.Name)))).To(Succeed())

				// top bar should show the pipeline is paused
				Eventually(page.Find(".top-bar.test.paused"), 10*time.Second).Should(BeFound())
			})
		})
	})

	Context("when manual triggering of the job is disabled", func() {
		var manualTriggerDisabledBuild atc.Build

		BeforeEach(func() {
			config := atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "job-manual-trigger-disabled",
					},
				},
			}

			byteConfig, err := yaml.Marshal(config)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, "0", byteConfig)
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			manualTriggerDisabledBuild, err = team.CreateJobBuild(pipelineName, "job-manual-trigger-disabled")
			Expect(err).NotTo(HaveOccurred())

			_, _, pipelineVersion, _, err := team.PipelineConfig(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			config = atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name:                 "job-manual-trigger-disabled",
						DisableManualTrigger: true,
					},
				},
			}

			byteConfig, err = yaml.Marshal(config)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, pipelineVersion, byteConfig)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should have a disabled button in the build details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))
			Eventually(page.Find("span.build-action"), 10*time.Second).Should(HaveAttribute("disabled", "true"))
		})

		It("should have a disabled button in the job details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))

			Eventually(page.Find("h1 a"), 10*time.Second).Should(BeFound())
			Expect(page.Find("h1 a").Click()).To(Succeed())
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled",
				teamName,
				pipelineName,
			))))

			Eventually(page.Find("button.build-action")).Should(BeFound())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", "true"))
		})
	})
})
<|endoftext|>


<commit_before>package timeout

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"runtime"
	"strings"
	"syscall"
	"testing"
	"time"
)

var (
	shellcmd  = "/bin/sh"
	shellflag = "-c"
)

const stubCmd = "testdata/stubcmd"

func init() {
	if runtime.GOOS == "windows" {
		shellcmd = "cmd"
		shellflag = "/c"
	}
	err := exec.Command("go", "build", "-o", stubCmd, "testdata/stubcmd.go").Run()
	if err != nil {
		panic(err)
	}
}

func TestRun(t *testing.T) {
	tio := &Timeout{
		Duration: 10 * time.Second,
		Cmd:      exec.Command(shellcmd, shellflag, "echo 1"),
	}
	_, stdout, stderr, err := tio.Run()

	if strings.TrimSpace(stdout) != "1" {
		t.Errorf("something wrong")
	}

	if stderr != "" {
		t.Errorf("something wrong")
	}

	if err != nil {
		t.Errorf("something wrong: %v", err)
	}
}

var isWin = runtime.GOOS == "windows"

func TestRunSimple(t *testing.T) {
	testCases := []struct {
		name           string
		duration       time.Duration
		killAfter      time.Duration
		cmd            *exec.Cmd
		signal         os.Signal
		preserveStatus bool
		expectedExit   int
		skipOnWin      bool
	}{
		{
			name:         "simple echo",
			duration:     time.Duration(0.1 * float64(time.Second)),
			cmd:          exec.Command(shellcmd, shellflag, "echo 1"),
			expectedExit: 0,
		},
		{
			name:         "timed out",
			cmd:          exec.Command(shellcmd, shellflag, fmt.Sprintf("%s -sleep 3", stubCmd)),
			duration:     1 * time.Second,
			signal:       os.Interrupt,
			expectedExit: 124,
		},
		{
			name:           "timed out with preserve status",
			cmd:            exec.Command(shellcmd, shellflag, fmt.Sprintf("%s -sleep 3", stubCmd)),
			duration:       time.Duration(0.1 * float64(time.Second)),
			preserveStatus: true,
			expectedExit:   128 + 15,
			skipOnWin:      true,
		},
		{
			name:           "preserve status (signal trapd)",
			cmd:            exec.Command(stubCmd, "-trap", "SIGTERM", "-trap-exit", "23", "-sleep", "3"),
			duration:       1 * time.Second,
			preserveStatus: true,
			expectedExit:   23,
			skipOnWin:      true,
		},
		{
			name:         "kill after",
			cmd:          exec.Command(stubCmd, "-trap", "SIGTERM", "-sleep", "3"),
			duration:     1 * time.Second,
			killAfter:    1 * time.Second,
			signal:       syscall.SIGTERM,
			expectedExit: exitKilled,
		},
		{
			name:           "trap sigterm but exited before kill after",
			cmd:            exec.Command(stubCmd, "-trap", "SIGTERM", "-sleep", "3"),
			duration:       1 * time.Second,
			killAfter:      5 * time.Second,
			signal:         syscall.SIGTERM,
			preserveStatus: true,
			expectedExit:   0,
		},
		{
			name:         "command cannnot be invoked",
			cmd:          exec.Command("testdata/dummy"),
			duration:     1 * time.Second,
			expectedExit: 126, // TODO cmd should return 125 on win
			skipOnWin:    true,
		},
		{
			name:         "command cannnot be invoked (command not found)",
			cmd:          exec.Command("testdata/command-not-found"),
			duration:     1 * time.Second,
			expectedExit: 127, // TODO cmd should return 125 on win
			skipOnWin:    true,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if tc.skipOnWin && isWin {
				t.Skipf("%s: skip on windows", tc.name)
			}
			tio := &Timeout{
				Duration:  tc.duration,
				KillAfter: tc.killAfter,
				Cmd:       tc.cmd,
				Signal:    tc.signal,
			}
			exit := tio.RunSimple(tc.preserveStatus)
			if exit != tc.expectedExit {
				t.Errorf("%s: expected exitcode: %d, but: %d", tc.name, tc.expectedExit, exit)
			}
		})
	}
}

func TestRunContext(t *testing.T) {
	expectedCode := 143
	if isWin {
		expectedCode = exitKilled
	}

	t.Run("cancel", func(t *testing.T) {
		tio := &Timeout{
			Duration: 3 * time.Second,
			Cmd:      exec.Command(stubCmd, "-sleep", "10"),
		}
		ctx, cancel := context.WithCancel(context.Background())
		go func() {
			time.Sleep(100 * time.Millisecond)
			cancel()
		}()
		st, err := tio.RunContext(ctx)
		if err != nil {
			t.Errorf("error should be nil but: %s", err)
		}
		expect := ExitStatus{
			Code:     expectedCode,
			Signaled: true,
			typ:      exitTypeCanceled,
			killed:   false,
		}
		if !reflect.DeepEqual(expect, *st) {
			t.Errorf("invalid exit status\n   out: %v\nexpect: %v", *st, expect)
		}
	})

	t.Run("with timeout", func(t *testing.T) {
		tio := &Timeout{
			Duration: 3 * time.Second,
			Cmd:      exec.Command(stubCmd, "-sleep", "10"),
		}
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		st, err := tio.RunContext(ctx)
		if err != nil {
			t.Errorf("error should be nil but: %s", err)
		}
		expect := ExitStatus{
			Code:     expectedCode,
			Signaled: true,
			typ:      exitTypeCanceled,
			killed:   false,
		}
		if !reflect.DeepEqual(expect, *st) {
			t.Errorf("invalid exit status\n   out: %v\nexpect: %v", *st, expect)
		}
	})

	t.Run("with timeout and signal trapped", func(t *testing.T) {
		if isWin {
			t.Skip("skip on windows")
		}
		tio := &Timeout{
			Duration:        3 * time.Second,
			Cmd:             exec.Command(stubCmd, "-sleep", "10", "-trap", "SIGTERM"),
			KillAfterCancel: time.Duration(10 * time.Millisecond),
		}
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		st, err := tio.RunContext(ctx)
		if err != nil {
			t.Errorf("error should be nil but: %s", err)
		}
		expect := ExitStatus{
			Code:     exitKilled,
			Signaled: true,
			typ:      exitTypeCanceled,
			killed:   true,
		}
		if !reflect.DeepEqual(expect, *st) {
			t.Errorf("invalid exit status\n   out: %v\nexpect: %v", *st, expect)
		}
	})
}
<commit_msg>fix for win<commit_after>package timeout

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"runtime"
	"strings"
	"syscall"
	"testing"
	"time"
)

var (
	shellcmd  = "/bin/sh"
	shellflag = "-c"
	stubCmd   = "./testdata/stubcmd"
)

func init() {
	if runtime.GOOS == "windows" {
		shellcmd = "cmd"
		shellflag = "/c"
		stubCmd = `.\testdata\stubcmd.exe`
	}
	err := exec.Command("go", "build", "-o", stubCmd, "testdata/stubcmd.go").Run()
	if err != nil {
		panic(err)
	}
}

func TestRun(t *testing.T) {
	tio := &Timeout{
		Duration: 10 * time.Second,
		Cmd:      exec.Command(shellcmd, shellflag, "echo 1"),
	}
	_, stdout, stderr, err := tio.Run()

	if strings.TrimSpace(stdout) != "1" {
		t.Errorf("something wrong")
	}

	if stderr != "" {
		t.Errorf("something wrong")
	}

	if err != nil {
		t.Errorf("something wrong: %v", err)
	}
}

var isWin = runtime.GOOS == "windows"

func TestRunSimple(t *testing.T) {
	testCases := []struct {
		name           string
		duration       time.Duration
		killAfter      time.Duration
		cmd            *exec.Cmd
		signal         os.Signal
		preserveStatus bool
		expectedExit   int
		skipOnWin      bool
	}{
		{
			name:         "simple echo",
			duration:     time.Duration(0.1 * float64(time.Second)),
			cmd:          exec.Command(shellcmd, shellflag, "echo 1"),
			expectedExit: 0,
		},
		{
			name:         "timed out",
			cmd:          exec.Command(shellcmd, shellflag, fmt.Sprintf("%s -sleep 3", stubCmd)),
			duration:     1 * time.Second,
			signal:       os.Interrupt,
			expectedExit: 124,
		},
		{
			name:           "timed out with preserve status",
			cmd:            exec.Command(shellcmd, shellflag, fmt.Sprintf("%s -sleep 3", stubCmd)),
			duration:       time.Duration(0.1 * float64(time.Second)),
			preserveStatus: true,
			expectedExit:   128 + 15,
			skipOnWin:      true,
		},
		{
			name:           "preserve status (signal trapd)",
			cmd:            exec.Command(stubCmd, "-trap", "SIGTERM", "-trap-exit", "23", "-sleep", "3"),
			duration:       1 * time.Second,
			preserveStatus: true,
			expectedExit:   23,
			skipOnWin:      true,
		},
		{
			name:         "kill after",
			cmd:          exec.Command(stubCmd, "-trap", "SIGTERM", "-sleep", "3"),
			duration:     1 * time.Second,
			killAfter:    1 * time.Second,
			signal:       syscall.SIGTERM,
			expectedExit: exitKilled,
		},
		{
			name:           "trap sigterm but exited before kill after",
			cmd:            exec.Command(stubCmd, "-trap", "SIGTERM", "-sleep", "3"),
			duration:       1 * time.Second,
			killAfter:      5 * time.Second,
			signal:         syscall.SIGTERM,
			preserveStatus: true,
			expectedExit:   0,
		},
		{
			name:         "command cannnot be invoked",
			cmd:          exec.Command("testdata/dummy"),
			duration:     1 * time.Second,
			expectedExit: 126, // TODO cmd should return 125 on win
			skipOnWin:    true,
		},
		{
			name:         "command cannnot be invoked (command not found)",
			cmd:          exec.Command("testdata/command-not-found"),
			duration:     1 * time.Second,
			expectedExit: 127, // TODO cmd should return 125 on win
			skipOnWin:    true,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if tc.skipOnWin && isWin {
				t.Skipf("%s: skip on windows", tc.name)
			}
			tio := &Timeout{
				Duration:  tc.duration,
				KillAfter: tc.killAfter,
				Cmd:       tc.cmd,
				Signal:    tc.signal,
			}
			exit := tio.RunSimple(tc.preserveStatus)
			if exit != tc.expectedExit {
				t.Errorf("%s: expected exitcode: %d, but: %d", tc.name, tc.expectedExit, exit)
			}
		})
	}
}

func TestRunContext(t *testing.T) {
	expect := ExitStatus{
		Code:     128 + int(syscall.SIGTERM),
		Signaled: true,
		typ:      exitTypeCanceled,
		killed:   false,
	}
	if isWin {
		expect = ExitStatus{
			Code:     1,
			Signaled: false,
			typ:      exitTypeCanceled,
			killed:   true,
		}
	}

	t.Run("cancel", func(t *testing.T) {
		tio := &Timeout{
			Duration: 3 * time.Second,
			Cmd:      exec.Command(stubCmd, "-sleep", "10"),
		}
		ctx, cancel := context.WithCancel(context.Background())
		go func() {
			time.Sleep(100 * time.Millisecond)
			cancel()
		}()
		st, err := tio.RunContext(ctx)
		if err != nil {
			t.Errorf("error should be nil but: %s", err)
		}
		if !reflect.DeepEqual(expect, *st) {
			t.Errorf("invalid exit status\n   out: %v\nexpect: %v", *st, expect)
		}
	})

	t.Run("with timeout", func(t *testing.T) {
		tio := &Timeout{
			Duration: 3 * time.Second,
			Cmd:      exec.Command(stubCmd, "-sleep", "10"),
		}
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		st, err := tio.RunContext(ctx)
		if err != nil {
			t.Errorf("error should be nil but: %s", err)
		}
		if !reflect.DeepEqual(expect, *st) {
			t.Errorf("invalid exit status\n   out: %v\nexpect: %v", *st, expect)
		}
	})

	t.Run("with timeout and signal trapped", func(t *testing.T) {
		if isWin {
			t.Skip("skip on windows")
		}
		tio := &Timeout{
			Duration:        3 * time.Second,
			Cmd:             exec.Command(stubCmd, "-sleep", "10", "-trap", "SIGTERM"),
			KillAfterCancel: time.Duration(10 * time.Millisecond),
		}
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		st, err := tio.RunContext(ctx)
		if err != nil {
			t.Errorf("error should be nil but: %s", err)
		}
		expect := ExitStatus{
			Code:     exitKilled,
			Signaled: true,
			typ:      exitTypeCanceled,
			killed:   true,
		}
		if !reflect.DeepEqual(expect, *st) {
			t.Errorf("invalid exit status\n   out: %v\nexpect: %v", *st, expect)
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/brocaar/chirpstack-gateway-bridge/cmd/chirpstack-gateway-bridge/cmd"
	paho "github.com/eclipse/paho.mqtt.golang"
	log "github.com/sirupsen/logrus"
)

type pahoLogWrapper struct {
	ln func(...interface{})
	f  func(string, ...interface{})
}

func (d pahoLogWrapper) Println(v ...interface{}) {
	d.ln(v...)
}

func (d pahoLogWrapper) Printf(format string, v ...interface{}) {
	d.f(format, v...)
}

func enableClientLogging() {
	l := log.WithField("module", "mqtt")
	paho.DEBUG = pahoLogWrapper{l.Debugln, l.Debugf}
	paho.ERROR = pahoLogWrapper{l.Errorln, l.Errorf}
	paho.WARN = pahoLogWrapper{l.Warningln, l.Warningf}
	paho.CRITICAL = pahoLogWrapper{l.Errorln, l.Errorf}
}

func init() {
	enableClientLogging()
}

var version string // set by the compiler

func main() {
	cmd.Execute(version)
}
<commit_msg>Add ns time precision to log output.<commit_after>package main

import (
	"time"

	"github.com/brocaar/chirpstack-gateway-bridge/cmd/chirpstack-gateway-bridge/cmd"
	paho "github.com/eclipse/paho.mqtt.golang"
	log "github.com/sirupsen/logrus"
)

type pahoLogWrapper struct {
	ln func(...interface{})
	f  func(string, ...interface{})
}

func (d pahoLogWrapper) Println(v ...interface{}) {
	d.ln(v...)
}

func (d pahoLogWrapper) Printf(format string, v ...interface{}) {
	d.f(format, v...)
}

func enableClientLogging() {
	l := log.WithField("module", "mqtt")
	paho.DEBUG = pahoLogWrapper{l.Debugln, l.Debugf}
	paho.ERROR = pahoLogWrapper{l.Errorln, l.Errorf}
	paho.WARN = pahoLogWrapper{l.Warningln, l.Warningf}
	paho.CRITICAL = pahoLogWrapper{l.Errorln, l.Errorf}
}

func init() {
	log.SetFormatter(&log.TextFormatter{
		TimestampFormat: time.RFC3339Nano,
	})

	enableClientLogging()
}

var version string // set by the compiler

func main() {
	cmd.Execute(version)
}
<|endoftext|>