<commit_before>// archive is package that helps create archives in a format that
// Atlas expects with its various upload endpoints.
package archive

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

//
type Archive struct {
	io.ReadCloser

	Size     int64
	Metadata map[string]string
}

// ArchiveOpts are the options for defining how the archive will be built.
type ArchiveOpts struct {
	// Exclude and Include are filters of files to include/exclude in
	// the archive when creating it from a directory. These filters should
	// be relative to the packaging directory and should be basic glob
	// patterns.
	Exclude []string
	Include []string

	// Extra is a mapping of extra files to include within the archive. The
	// key should be the path within the archive and the value should be
	// an absolute path to the file to put into the archive. These extra
	// files will override any other files in the archive.
	Extra map[string]string

	// VCS, if true, will detect and use a VCS system to determine what
	// files to include the archive.
	VCS bool
}

// IsSet says whether any options were set.
func (o *ArchiveOpts) IsSet() bool {
	return len(o.Exclude) > 0 || len(o.Include) > 0 || o.VCS
}

// CreateArchive takes the given path and ArchiveOpts and archives it.
//
// The archive will be fully completed and put into a temporary file.
// This must be done to retrieve the content length of the archive which
// is needed for almost all operations involving archives with Atlas. Because
// of this, sufficient disk space will be required to buffer the archive.
func CreateArchive(path string, opts *ArchiveOpts) (*Archive, error) {
	log.Printf("[INFO] creating archive from %s", path)

	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// Direct file paths cannot have archive options
	if !fi.IsDir() && opts.IsSet() {
		return nil, fmt.Errorf(
			"options such as exclude, include, and VCS can't be set when " +
				"the path is a file.")
	}

	if fi.IsDir() {
		return archiveDir(path, opts)
	} else {
		return archiveFile(path, opts)
	}
}

func archiveFile(path string, opts *ArchiveOpts) (*Archive, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	if _, err := gzip.NewReader(f); err == nil {
		// Reset the read offset for future reading
		if _, err := f.Seek(0, 0); err != nil {
			f.Close()
			return nil, err
		}

		// Get the file info for the size
		fi, err := f.Stat()
		if err != nil {
			f.Close()
			return nil, err
		}

		// This is a gzip file, let it through.
		return &Archive{ReadCloser: f, Size: fi.Size()}, nil
	}

	// Close the file, no use for it anymore
	f.Close()

	// We have a single file that is not gzipped. Compress it.
	path, err = filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	// Act like we're compressing a directory, but only include this one
	// file.
	return archiveDir(filepath.Dir(path), &ArchiveOpts{
		Include: []string{filepath.Base(path)},
	})
}

func archiveDir(root string, opts *ArchiveOpts) (*Archive, error) {
	var vcsInclude []string
	var metadata map[string]string
	if opts.VCS {
		var err error

		if err = vcsPreflight(root); err != nil {
			return nil, err
		}

		vcsInclude, err = vcsFiles(root)
		if err != nil {
			return nil, err
		}

		metadata, err = vcsMetadata(root)
		if err != nil {
			return nil, err
		}
	}

	// Create the temporary file that we'll send the archive data to.
	archiveF, err := ioutil.TempFile("", "atlas-archive")
	if err != nil {
		return nil, err
	}

	// Create the wrapper for the result which will automatically
	// remove the temporary file on close.
	archiveWrapper := &readCloseRemover{F: archiveF}

	// Buffer the writer so that we can push as much data to disk at
	// a time as possible. 4M should be good.
	bufW := bufio.NewWriterSize(archiveF, 4096*1024)

	// Gzip compress all the output data
	gzipW := gzip.NewWriter(bufW)

	// Tar the file contents
	tarW := tar.NewWriter(gzipW)

	// Build the function that'll do all the compression
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Get the relative path from the path since it contains the root
		// plus the path.
		subpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if subpath == "." {
			return nil
		}

		// If we have a list of VCS files, check that first
		skip := false
		if len(vcsInclude) > 0 {
			skip = true
			for _, f := range vcsInclude {
				if f == subpath {
					skip = false
					break
				}

				if info.IsDir() && strings.HasPrefix(f, subpath+"/") {
					skip = false
					break
				}
			}
		}

		// If include is present, we only include what is listed
		if len(opts.Include) > 0 {
			skip = true
			for _, include := range opts.Include {
				match, err := filepath.Match(include, subpath)
				if err != nil {
					return err
				}
				if match {
					skip = false
					break
				}
			}
		}

		// If exclude, it is one last gate to excluding files
		for _, exclude := range opts.Exclude {
			match, err := filepath.Match(exclude, subpath)
			if err != nil {
				return err
			}
			if match {
				skip = true
				break
			}
		}

		// If we have to skip this file, then skip it, properly skipping
		// children if we're a directory.
		if skip {
			if info.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = subpath
		if info.IsDir() {
			header.Name += "/"
		}

		// Write the header first to the archive.
		if err := tarW.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"failed opening file '%s' to write compressed archive.", path)
		}
		defer f.Close()

		if _, err = io.Copy(tarW, f); err != nil {
			return fmt.Errorf(
				"failed copying file to archive: %s", path)
		}

		return nil
	}

	// First, walk the path and do the normal files
	werr := filepath.Walk(root, walkFn)
	if werr == nil {
		// If that succeeded, handle the extra files
		werr = copyExtras(tarW, opts.Extra)
	}

	// Attempt to close all the things. If we get an error on the way
	// and we haven't had an error yet, then record that as the critical
	// error. But we still try to close everything.

	// Close the tar writer
	if err := tarW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Close the gzip writer
	if err := gzipW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Flush the buffer
	if err := bufW.Flush(); err != nil && werr == nil {
		werr = err
	}

	// If we had an error, then close the file (removing it) and
	// return the error.
	if werr != nil {
		archiveWrapper.Close()
		return nil, werr
	}

	// Seek to the beginning
	if _, err := archiveWrapper.F.Seek(0, 0); err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	// Get the file information so we can get the size
	fi, err := archiveWrapper.F.Stat()
	if err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	return &Archive{
		ReadCloser: archiveWrapper,
		Size:       fi.Size(),
		Metadata:   metadata,
	}, nil
}

func copyExtras(w *tar.Writer, extra map[string]string) error {
	for entry, path := range extra {
		info, err := os.Stat(path)
		if err != nil {
			return err
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = entry

		// Write the header first to the archive.
		if err := w.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"failed opening file '%s' to write compressed archive.", path)
		}

		_, err = io.Copy(w, f)
		f.Close()
		if err != nil {
			return fmt.Errorf(
				"failed copying file to archive: %s", path)
		}
	}

	return nil
}

// readCloseRemover is an io.ReadCloser implementation that will remove
// the file on Close(). We use this to clean up our temporary file for
// the archive.
type readCloseRemover struct {
	F *os.File
}

func (r *readCloseRemover) Read(p []byte) (int, error) {
	return r.F.Read(p)
}

func (r *readCloseRemover) Close() error {
	// First close the file
	err := r.F.Close()

	// Next make sure to remove it, or at least try, regardless of error
	// above.
	os.Remove(r.F.Name())

	return err
}
<commit_msg>archive: comments<commit_after>// archive is package that helps create archives in a format that
// Atlas expects with its various upload endpoints.
package archive

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// Archive is the resulting archive. The archive data is generally streamed
// so the io.ReadCloser can be used to backpressure the archive progress
// and avoid memory pressure.
type Archive struct {
	io.ReadCloser

	Size     int64
	Metadata map[string]string
}

// ArchiveOpts are the options for defining how the archive will be built.
type ArchiveOpts struct {
	// Exclude and Include are filters of files to include/exclude in
	// the archive when creating it from a directory. These filters should
	// be relative to the packaging directory and should be basic glob
	// patterns.
	Exclude []string
	Include []string

	// Extra is a mapping of extra files to include within the archive. The
	// key should be the path within the archive and the value should be
	// an absolute path to the file to put into the archive. These extra
	// files will override any other files in the archive.
	Extra map[string]string

	// VCS, if true, will detect and use a VCS system to determine what
	// files to include the archive.
	VCS bool
}

// IsSet says whether any options were set.
func (o *ArchiveOpts) IsSet() bool {
	return len(o.Exclude) > 0 || len(o.Include) > 0 || o.VCS
}

// CreateArchive takes the given path and ArchiveOpts and archives it.
//
// The archive will be fully completed and put into a temporary file.
// This must be done to retrieve the content length of the archive which
// is needed for almost all operations involving archives with Atlas. Because
// of this, sufficient disk space will be required to buffer the archive.
func CreateArchive(path string, opts *ArchiveOpts) (*Archive, error) {
	log.Printf("[INFO] creating archive from %s", path)

	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// Direct file paths cannot have archive options
	if !fi.IsDir() && opts.IsSet() {
		return nil, fmt.Errorf(
			"options such as exclude, include, and VCS can't be set when " +
				"the path is a file.")
	}

	if fi.IsDir() {
		return archiveDir(path, opts)
	} else {
		return archiveFile(path, opts)
	}
}

func archiveFile(path string, opts *ArchiveOpts) (*Archive, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	if _, err := gzip.NewReader(f); err == nil {
		// Reset the read offset for future reading
		if _, err := f.Seek(0, 0); err != nil {
			f.Close()
			return nil, err
		}

		// Get the file info for the size
		fi, err := f.Stat()
		if err != nil {
			f.Close()
			return nil, err
		}

		// This is a gzip file, let it through.
		return &Archive{ReadCloser: f, Size: fi.Size()}, nil
	}

	// Close the file, no use for it anymore
	f.Close()

	// We have a single file that is not gzipped. Compress it.
	path, err = filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	// Act like we're compressing a directory, but only include this one
	// file.
	return archiveDir(filepath.Dir(path), &ArchiveOpts{
		Include: []string{filepath.Base(path)},
	})
}

func archiveDir(root string, opts *ArchiveOpts) (*Archive, error) {
	var vcsInclude []string
	var metadata map[string]string
	if opts.VCS {
		var err error

		if err = vcsPreflight(root); err != nil {
			return nil, err
		}

		vcsInclude, err = vcsFiles(root)
		if err != nil {
			return nil, err
		}

		metadata, err = vcsMetadata(root)
		if err != nil {
			return nil, err
		}
	}

	// Create the temporary file that we'll send the archive data to.
	archiveF, err := ioutil.TempFile("", "atlas-archive")
	if err != nil {
		return nil, err
	}

	// Create the wrapper for the result which will automatically
	// remove the temporary file on close.
	archiveWrapper := &readCloseRemover{F: archiveF}

	// Buffer the writer so that we can push as much data to disk at
	// a time as possible. 4M should be good.
	bufW := bufio.NewWriterSize(archiveF, 4096*1024)

	// Gzip compress all the output data
	gzipW := gzip.NewWriter(bufW)

	// Tar the file contents
	tarW := tar.NewWriter(gzipW)

	// Build the function that'll do all the compression
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Get the relative path from the path since it contains the root
		// plus the path.
		subpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if subpath == "." {
			return nil
		}

		// If we have a list of VCS files, check that first
		skip := false
		if len(vcsInclude) > 0 {
			skip = true
			for _, f := range vcsInclude {
				if f == subpath {
					skip = false
					break
				}

				if info.IsDir() && strings.HasPrefix(f, subpath+"/") {
					skip = false
					break
				}
			}
		}

		// If include is present, we only include what is listed
		if len(opts.Include) > 0 {
			skip = true
			for _, include := range opts.Include {
				match, err := filepath.Match(include, subpath)
				if err != nil {
					return err
				}
				if match {
					skip = false
					break
				}
			}
		}

		// If exclude, it is one last gate to excluding files
		for _, exclude := range opts.Exclude {
			match, err := filepath.Match(exclude, subpath)
			if err != nil {
				return err
			}
			if match {
				skip = true
				break
			}
		}

		// If we have to skip this file, then skip it, properly skipping
		// children if we're a directory.
		if skip {
			if info.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = subpath
		if info.IsDir() {
			header.Name += "/"
		}

		// Write the header first to the archive.
		if err := tarW.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"failed opening file '%s' to write compressed archive.", path)
		}
		defer f.Close()

		if _, err = io.Copy(tarW, f); err != nil {
			return fmt.Errorf(
				"failed copying file to archive: %s", path)
		}

		return nil
	}

	// First, walk the path and do the normal files
	werr := filepath.Walk(root, walkFn)
	if werr == nil {
		// If that succeeded, handle the extra files
		werr = copyExtras(tarW, opts.Extra)
	}

	// Attempt to close all the things. If we get an error on the way
	// and we haven't had an error yet, then record that as the critical
	// error. But we still try to close everything.

	// Close the tar writer
	if err := tarW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Close the gzip writer
	if err := gzipW.Close(); err != nil && werr == nil {
		werr = err
	}

	// Flush the buffer
	if err := bufW.Flush(); err != nil && werr == nil {
		werr = err
	}

	// If we had an error, then close the file (removing it) and
	// return the error.
	if werr != nil {
		archiveWrapper.Close()
		return nil, werr
	}

	// Seek to the beginning
	if _, err := archiveWrapper.F.Seek(0, 0); err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	// Get the file information so we can get the size
	fi, err := archiveWrapper.F.Stat()
	if err != nil {
		archiveWrapper.Close()
		return nil, err
	}

	return &Archive{
		ReadCloser: archiveWrapper,
		Size:       fi.Size(),
		Metadata:   metadata,
	}, nil
}

func copyExtras(w *tar.Writer, extra map[string]string) error {
	for entry, path := range extra {
		info, err := os.Stat(path)
		if err != nil {
			return err
		}

		// Read the symlink target. We don't track the error because
		// it doesn't matter if there is an error.
		target, _ := os.Readlink(path)

		// Build the file header for the tar entry
		header, err := tar.FileInfoHeader(info, target)
		if err != nil {
			return fmt.Errorf(
				"failed creating archive header: %s", path)
		}

		// Modify the header to properly be the full subpath
		header.Name = entry

		// Write the header first to the archive.
		if err := w.WriteHeader(header); err != nil {
			return fmt.Errorf(
				"failed writing archive header: %s", path)
		}

		// If it is a directory, then we're done (no body to write)
		if info.IsDir() {
			return nil
		}

		// Open the target file to write the data
		f, err := os.Open(path)
		if err != nil {
			return fmt.Errorf(
				"failed opening file '%s' to write compressed archive.", path)
		}

		_, err = io.Copy(w, f)
		f.Close()
		if err != nil {
			return fmt.Errorf(
				"failed copying file to archive: %s", path)
		}
	}

	return nil
}

// readCloseRemover is an io.ReadCloser implementation that will remove
// the file on Close(). We use this to clean up our temporary file for
// the archive.
type readCloseRemover struct {
	F *os.File
}

func (r *readCloseRemover) Read(p []byte) (int, error) {
	return r.F.Read(p)
}

func (r *readCloseRemover) Close() error {
	// First close the file
	err := r.F.Close()

	// Next make sure to remove it, or at least try, regardless of error
	// above.
	os.Remove(r.F.Name())

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sync"

	"github.com/cheggaaa/pb"
	_ "github.com/mattn/go-sqlite3"
)

const (
	thumbsDir      = "public/thumbs"
	bigThumbSize   = "1000"
	smallThumbSize = "200"
	workers        = 4 // min: 1
)

func generateSmallThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_small.jpg", identifier))

	absThumbPath, err := filepath.Abs(thumbPath)
	if err != nil {
		return
	}

	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"vipsthumbnail", photoPath,
			"--rotate",
			"--size", smallThumbSize,
			"--crop",
			"--interpolator", "bicubic",
			"--output", absThumbPath+"[Q=97,no_subsample,strip]").Run()
	}

	return
}

func generateBigThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_big.jpg", identifier))

	absThumbPath, err := filepath.Abs(thumbPath)
	if err != nil {
		return
	}

	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"vipsthumbnail", photoPath,
			"--rotate",
			"--size", bigThumbSize,
			"--interpolator", "bicubic",
			"--output", absThumbPath+"[Q=97,no_subsample,strip]").Run()
	}

	return
}

func generateThumbsImpl(photoPath string) (err error) {
	identifier := fmt.Sprintf("%x", md5.Sum([]byte(photoPath)))

	bigThumbPath, err := generateBigThumb(photoPath, identifier)
	if err == nil { // success
		_, err = generateSmallThumb(bigThumbPath, identifier)
	}

	return
}

func generateThumbs(ch chan string, wg *sync.WaitGroup, bar *pb.ProgressBar) {
	defer wg.Done()

	for photoPath := range ch {
		generateThumbsImpl(photoPath)
		bar.Increment()
	}
}

func main() {
	var photosCount int

	if workers < 1 {
		log.Fatal("number of workers must be at least 1")
	}

	db, err := sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	rows, err := db.Query(`
	SELECT path FROM photos
	JOIN sets ON photos.set_id = sets.id
	ORDER BY sets.taken_at DESC, photos.taken_at ASC
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	err = db.QueryRow("SELECT COUNT(*) FROM photos").Scan(&photosCount)
	if err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(thumbsDir, os.ModeDir|0755); err != nil {
		log.Fatal(err)
	}

	ch := make(chan string)
	wg := sync.WaitGroup{}
	bar := pb.StartNew(photosCount)

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go generateThumbs(ch, &wg, bar)
	}

	for rows.Next() {
		var photoPath string
		rows.Scan(&photoPath)
		ch <- photoPath
	}

	close(ch)
	wg.Wait()

	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Generate small thumb from original if big thumb failed<commit_after>package main

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sync"

	"github.com/cheggaaa/pb"
	_ "github.com/mattn/go-sqlite3"
)

const (
	thumbsDir      = "public/thumbs"
	bigThumbSize   = "1000"
	smallThumbSize = "200"
	workers        = 4 // min: 1
)

func generateSmallThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_small.jpg", identifier))

	absThumbPath, err := filepath.Abs(thumbPath)
	if err != nil {
		return
	}

	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"vipsthumbnail", photoPath,
			"--rotate",
			"--size", smallThumbSize,
			"--crop",
			"--interpolator", "bicubic",
			"--output", absThumbPath+"[Q=97,no_subsample,strip]").Run()
	}

	return
}

func generateBigThumb(photoPath, identifier string) (thumbPath string, err error) {
	thumbPath = path.Join(thumbsDir, fmt.Sprintf("%s_big.jpg", identifier))

	absThumbPath, err := filepath.Abs(thumbPath)
	if err != nil {
		return
	}

	if _, err = os.Stat(thumbPath); os.IsNotExist(err) { // file does not exist
		err = exec.Command(
			"vipsthumbnail", photoPath,
			"--rotate",
			"--size", bigThumbSize,
			"--interpolator", "bicubic",
			"--output", absThumbPath+"[Q=97,no_subsample,strip]").Run()
	}

	return
}

func generateThumbsImpl(photoPath string) (err error) {
	identifier := fmt.Sprintf("%x", md5.Sum([]byte(photoPath)))

	bigThumbPath, err := generateBigThumb(photoPath, identifier)
	if err == nil { // success
		_, err = generateSmallThumb(bigThumbPath, identifier)
	} else { // create from original photo since big thumb failed
		_, err = generateSmallThumb(photoPath, identifier)
	}

	return
}

func generateThumbs(ch chan string, wg *sync.WaitGroup, bar *pb.ProgressBar) {
	defer wg.Done()

	for photoPath := range ch {
		generateThumbsImpl(photoPath)
		bar.Increment()
	}
}

func main() {
	var photosCount int

	if workers < 1 {
		log.Fatal("number of workers must be at least 1")
	}

	db, err := sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	rows, err := db.Query(`
	SELECT path FROM photos
	JOIN sets ON photos.set_id = sets.id
	ORDER BY sets.taken_at DESC, photos.taken_at ASC
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	err = db.QueryRow("SELECT COUNT(*) FROM photos").Scan(&photosCount)
	if err != nil {
		log.Fatal(err)
	}

	if err := os.MkdirAll(thumbsDir, os.ModeDir|0755); err != nil {
		log.Fatal(err)
	}

	ch := make(chan string)
	wg := sync.WaitGroup{}
	bar := pb.StartNew(photosCount)

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go generateThumbs(ch, &wg, bar)
	}

	for rows.Next() {
		var photoPath string
		rows.Scan(&photoPath)
		ch <- photoPath
	}

	close(ch)
	wg.Wait()

	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"k8s.io/kubernetes/pkg/volume"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/blockstorage/v1/volumes"
	"github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"github.com/rackspace/gophercloud/pagination"

	"github.com/golang/glog"
)

// Attaches given cinder volume to the compute running kubelet
func (os *OpenStack) AttachDisk(instanceID string, diskName string) (string, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return "", err
	}

	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			glog.V(4).Infof("Disk: %q is already attached to compute: %q", diskName, instanceID)
			return disk.ID, nil
		} else {
			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", diskName, disk.Attachments[0]["server_id"])
			glog.Errorf(errMsg)
			return "", errors.New(errMsg)
		}
	}
	// add read only flag here if possible spothanis
	_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
		VolumeID: disk.ID,
	}).Extract()
	if err != nil {
		glog.Errorf("Failed to attach %s volume to %s compute", diskName, instanceID)
		return "", err
	}
	glog.V(2).Infof("Successfully attached %s volume to %s compute", diskName, instanceID)
	return disk.ID, nil
}

// Detaches given cinder volume from the compute running kubelet
func (os *OpenStack) DetachDisk(instanceID string, partialDiskId string) error {
	disk, err := os.getVolume(partialDiskId)
	if err != nil {
		return err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		// This is a blocking call and effects kubelet's performance directly.
		// We should consider kicking it out into a separate routine, if it is bad.
		err = volumeattach.Delete(cClient, instanceID, disk.ID).ExtractErr()
		if err != nil {
			glog.Errorf("Failed to delete volume %s from compute %s attached %v", disk.ID, instanceID, err)
			return err
		}
		glog.V(2).Infof("Successfully detached volume: %s from compute: %s", disk.ID, instanceID)
	} else {
		errMsg := fmt.Sprintf("Disk: %s has no attachments or is not attached to compute: %s", disk.Name, instanceID)
		glog.Errorf(errMsg)
		return errors.New(errMsg)
	}
	return nil
}

// Takes a partial/full disk id or diskname
func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) {
	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	var volume volumes.Volume
	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return volume, err
	}

	err = volumes.List(sClient, nil).EachPage(func(page pagination.Page) (bool, error) {
		vols, err := volumes.ExtractVolumes(page)
		if err != nil {
			glog.Errorf("Failed to extract volumes: %v", err)
			return false, err
		} else {
			for _, v := range vols {
				glog.V(4).Infof("%s %s %v", v.ID, v.Name, v.Attachments)
				if v.Name == diskName || strings.Contains(v.ID, diskName) {
					volume = v
					return true, nil
				}
			}
		}
		// if it reached here then no disk with the given name was found.
		errmsg := fmt.Sprintf("Unable to find disk: %s in region %s", diskName, os.region)
		return false, errors.New(errmsg)
	})
	if err != nil {
		glog.Errorf("Error occurred getting volume: %s", diskName)
		return volume, err
	}
	return volume, err
}

// Create a volume of given size (in GiB)
func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (volumeName string, err error) {

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return "", err
	}

	opts := volumes.CreateOpts{
		Name:         name,
		Size:         size,
		VolumeType:   vtype,
		Availability: availability,
	}
	if tags != nil {
		opts.Metadata = *tags
	}
	vol, err := volumes.Create(sClient, opts).Extract()
	if err != nil {
		glog.Errorf("Failed to create a %d GB volume: %v", size, err)
		return "", err
	}
	glog.Infof("Created volume %v", vol.ID)
	return vol.ID, err
}

// GetDevicePath returns the path of an attached block storage volume, specified by its id.
func (os *OpenStack) GetDevicePath(diskId string) string {
	files, _ := ioutil.ReadDir("/dev/disk/by-id/")
	for _, f := range files {
		if strings.Contains(f.Name(), "virtio-") {
			devid_prefix := f.Name()[len("virtio-"):len(f.Name())]
			if strings.Contains(diskId, devid_prefix) {
				glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name()))
				return path.Join("/dev/disk/by-id/", f.Name())
			}
		}
	}
	glog.Warningf("Failed to find device for the diskid: %q\n", diskId)
	return ""
}

func (os *OpenStack) DeleteVolume(volumeName string) error {
	used, err := os.diskIsUsed(volumeName)
	if err != nil {
		return err
	}
	if used {
		msg := fmt.Sprintf("Cannot delete the volume %q, it's still attached to a node", volumeName)
		return volume.NewDeletedVolumeInUseError(msg)
	}

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return err
	}
	err = volumes.Delete(sClient, volumeName).ExtractErr()
	if err != nil {
		glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
	}
	return err
}

// Get device path of attached volume to the compute running kubelet
func (os *OpenStack) GetAttachmentDiskPath(instanceID string, diskName string) (string, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			// Attachment[0]["device"] points to the device path
			// see http://developer.openstack.org/api-ref-blockstorage-v1.html
			return disk.Attachments[0]["device"].(string), nil
		} else {
			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", diskName, disk.Attachments[0]["server_id"])
			glog.Errorf(errMsg)
			return "", errors.New(errMsg)
		}
	}
	return "", fmt.Errorf("volume %s is not attached to %s", diskName, instanceID)
}

// query if a volume is attached to a compute instance
func (os *OpenStack) DiskIsAttached(diskName, instanceID string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		return true, nil
	}
	return false, nil
}

// diskIsUsed returns true a disk is attached to any node.
func (os *OpenStack) diskIsUsed(diskName string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 {
		return true, nil
	}
	return false, nil
}
<commit_msg>Add sync state loop in master's volume reconciler<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"k8s.io/kubernetes/pkg/volume"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/blockstorage/v1/volumes"
	"github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"github.com/rackspace/gophercloud/pagination"

	"github.com/golang/glog"
)

// Attaches given cinder volume to the compute running kubelet
func (os *OpenStack) AttachDisk(instanceID string, diskName string) (string, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return "", err
	}

	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			glog.V(4).Infof("Disk: %q is already attached to compute: %q", diskName, instanceID)
			return disk.ID, nil
		} else {
			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", diskName, disk.Attachments[0]["server_id"])
			glog.Errorf(errMsg)
			return "", errors.New(errMsg)
		}
	}
	// add read only flag here if possible spothanis
	_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
		VolumeID: disk.ID,
	}).Extract()
	if err != nil {
		glog.Errorf("Failed to attach %s volume to %s compute", diskName, instanceID)
		return "", err
	}
	glog.V(2).Infof("Successfully attached %s volume to %s compute", diskName, instanceID)
	return disk.ID, nil
}

// Detaches given cinder volume from the compute running kubelet
func (os *OpenStack) DetachDisk(instanceID string, partialDiskId string) error {
	disk, err := os.getVolume(partialDiskId)
	if err != nil {
		return err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		// This is a blocking call and effects kubelet's performance directly.
		// We should consider kicking it out into a separate routine, if it is bad.
		err = volumeattach.Delete(cClient, instanceID, disk.ID).ExtractErr()
		if err != nil {
			glog.Errorf("Failed to delete volume %s from compute %s attached %v", disk.ID, instanceID, err)
			return err
		}
		glog.V(2).Infof("Successfully detached volume: %s from compute: %s", disk.ID, instanceID)
	} else {
		errMsg := fmt.Sprintf("Disk: %s has no attachments or is not attached to compute: %s", disk.Name, instanceID)
		glog.Errorf(errMsg)
		return errors.New(errMsg)
	}
	return nil
}

// Takes a partial/full disk id or diskname
func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) {
	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	var volume volumes.Volume
	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return volume, err
	}

	err = volumes.List(sClient, nil).EachPage(func(page pagination.Page) (bool, error) {
		vols, err := volumes.ExtractVolumes(page)
		if err != nil {
			glog.Errorf("Failed to extract volumes: %v", err)
			return false, err
		} else {
			for _, v := range vols {
				glog.V(4).Infof("%s %s %v", v.ID, v.Name, v.Attachments)
				if v.Name == diskName || strings.Contains(v.ID, diskName) {
					volume = v
					return true, nil
				}
			}
		}
		// if it reached here then no disk with the given name was found.
		errmsg := fmt.Sprintf("Unable to find disk: %s in region %s", diskName, os.region)
		return false, errors.New(errmsg)
	})
	if err != nil {
		glog.Errorf("Error occurred getting volume: %s", diskName)
		return volume, err
	}
	return volume, err
}

// Create a volume of given size (in GiB)
func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (volumeName string, err error) {

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return "", err
	}

	opts := volumes.CreateOpts{
		Name:         name,
		Size:         size,
		VolumeType:   vtype,
		Availability: availability,
	}
	if tags != nil {
		opts.Metadata = *tags
	}
	vol, err := volumes.Create(sClient, opts).Extract()
	if err != nil {
		glog.Errorf("Failed to create a %d GB volume: %v", size, err)
		return "", err
	}
	glog.Infof("Created volume %v", vol.ID)
	return vol.ID, err
}

// GetDevicePath returns the path of an attached block storage volume, specified by its id.
func (os *OpenStack) GetDevicePath(diskId string) string {
	files, _ := ioutil.ReadDir("/dev/disk/by-id/")
	for _, f := range files {
		if strings.Contains(f.Name(), "virtio-") {
			devid_prefix := f.Name()[len("virtio-"):len(f.Name())]
			if strings.Contains(diskId, devid_prefix) {
				glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name()))
				return path.Join("/dev/disk/by-id/", f.Name())
			}
		}
	}
	glog.Warningf("Failed to find device for the diskid: %q\n", diskId)
	return ""
}

func (os *OpenStack) DeleteVolume(volumeName string) error {
	used, err := os.diskIsUsed(volumeName)
	if err != nil {
		return err
	}
	if used {
		msg := fmt.Sprintf("Cannot delete the volume %q, it's still attached to a node", volumeName)
		return volume.NewDeletedVolumeInUseError(msg)
	}

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return err
	}
	err = volumes.Delete(sClient, volumeName).ExtractErr()
	if err != nil {
		glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
	}
	return err
}

// Get device path of attached volume to the compute running kubelet
func (os *OpenStack) GetAttachmentDiskPath(instanceID string, diskName string) (string, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			// Attachment[0]["device"] points to the device path
			// see http://developer.openstack.org/api-ref-blockstorage-v1.html
			return disk.Attachments[0]["device"].(string), nil
		} else {
			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", diskName, disk.Attachments[0]["server_id"])
			glog.Errorf(errMsg)
			return "", errors.New(errMsg)
		}
	}
	return "", fmt.Errorf("volume %s is not attached to %s", diskName, instanceID)
}

// query if a volume is attached to a compute instance
func (os *OpenStack) DiskIsAttached(diskName, instanceID string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		return true, nil
	}
	return false, nil
}

// query if a list of volumes are attached to a compute instance
func (os *OpenStack) DisksAreAttached(diskNames []string, instanceID string) (map[string]bool, error) {
	attached := make(map[string]bool)
	for _, diskName := range diskNames {
		attached[diskName] = false
	}
	for _, diskName := range diskNames {
		disk, err := os.getVolume(diskName)
		if err != nil {
			continue
		}
		if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
			attached[diskName] = true
		}
	}
	return attached, nil
}

// diskIsUsed returns true a disk is attached to any node.
func (os *OpenStack) diskIsUsed(diskName string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 {
		return true, nil
	}
	return false, nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"github.com/netlify/gotrue/models"
)

const defaultPerPage = 50

func calculateTotalPages(perPage, total uint64) uint64 {
	pages := total / perPage
	if total%perPage > 0 {
		return pages + 1
	}
	return pages
}

func addPaginationHeaders(w http.ResponseWriter, r *http.Request, p *models.Pagination) {
	totalPages := calculateTotalPages(p.PerPage, p.Count)
	url, _ := url.ParseRequestURI(r.URL.String())
	query := url.Query()
	header := ""
	if totalPages > p.Page {
		query.Set("page", fmt.Sprintf("%v", p.Page+1))
		url.RawQuery = query.Encode()
		header += "<" + url.String() + ">; rel=\"next\", "
	}
	query.Set("page", fmt.Sprintf("%v", totalPages))
	url.RawQuery = query.Encode()
	header += "<" + url.String() + ">; rel=\"last\""

	w.Header().Add("Link", header)
	w.Header().Add("X-Total-Count", fmt.Sprintf("%v", p.Count))
}

func paginate(r *http.Request) (*models.Pagination, error) {
	params := r.URL.Query()
	queryPage := params.Get("page")
	queryPerPage := params.Get("per_page")
	var page uint64 = 1
	var perPage uint64 = defaultPerPage
	var err error
	if queryPage != "" {
		page, err = strconv.ParseUint(queryPage, 10, 64)
		if err != nil {
			return nil, err
		}
	}
	if queryPerPage != "" {
		perPage, err = strconv.ParseUint(queryPerPage, 10, 64)
		if err != nil {
			return nil, err
		}
	}

	return &models.Pagination{
		Page:    page,
		PerPage: perPage,
	}, nil
}
<commit_msg>add prev pagination header<commit_after>package api

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"github.com/netlify/gotrue/models"
)

const defaultPerPage = 50

func calculateTotalPages(perPage, total uint64) uint64 {
	pages := total / perPage
	if total%perPage > 0 {
		return pages + 1
	}
	return pages
}

func addPaginationHeaders(w http.ResponseWriter, r *http.Request, p *models.Pagination) {
	totalPages := calculateTotalPages(p.PerPage, p.Count)
	url, _ := url.ParseRequestURI(r.URL.String())
	query := url.Query()
	header := ""
	if totalPages > p.Page {
		query.Set("page", fmt.Sprintf("%v", p.Page+1))
		url.RawQuery = query.Encode()
		header += "<" + url.String() + ">; rel=\"next\", "
	}
	if p.Page > 1 {
		query.Set("page", fmt.Sprintf("%v", p.Page-1))
		url.RawQuery = query.Encode()
		header += "<" + url.String() + ">; rel=\"prev\", "
	}
	query.Set("page", fmt.Sprintf("%v", totalPages))
	url.RawQuery = query.Encode()
	header += "<" + url.String() + ">; rel=\"last\""

	w.Header().Add("Link", header)
	w.Header().Add("X-Total-Count", fmt.Sprintf("%v", p.Count))
}

func paginate(r *http.Request) (*models.Pagination, error) {
	params := r.URL.Query()
	queryPage := params.Get("page")
	queryPerPage := params.Get("per_page")
	var page uint64 = 1
	var perPage uint64 = defaultPerPage
	var err error
	if queryPage != "" {
		page, err = strconv.ParseUint(queryPage, 10, 64)
		if err != nil {
			return nil, err
		}
	}
	if queryPerPage != "" {
		perPage, err = strconv.ParseUint(queryPerPage, 10, 64)
		if err != nil {
			return nil, err
		}
	}

	return &models.Pagination{
		Page:    page,
		PerPage: perPage,
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/alecthomas/jsonschema"

	vegeta "github.com/tsenart/vegeta/lib"
)

func main() {
	types := map[string]interface{}{
		"Target": &vegeta.Target{},
	}

	valid := strings.Join(keys(types), ", ")

	fs := flag.NewFlagSet("jsonschema", flag.ExitOnError)
	typ := fs.String("type", "", fmt.Sprintf("Vegeta type to generate a JSON schema for [%s]", valid))
	out := fs.String("output", "stdout", "Output file")

	fs.Parse(os.Args[1:])

	t, ok := types[*typ]
	if !ok {
		die("invalid type %q not in [%s]", *typ, valid)
	}

	schema, err := json.MarshalIndent(jsonschema.Reflect(t), "", "  ")
	if err != nil {
		die("%s", err)
	}

	switch *out {
	case "stdout":
		_, err = os.Stdout.Write(schema)
	default:
		err = ioutil.WriteFile(*out, schema, 0644)
	}

	if err != nil {
		die("%s", err)
	}
}

func die(s string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, s, args...)
	os.Exit(1)
}

func keys(types map[string]interface{}) (ks []string) {
	for k := range types {
		ks = append(ks, k)
	}
	return ks
}
<commit_msg>jsonschema: Exit on fs.Parse error<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/alecthomas/jsonschema"

	vegeta "github.com/tsenart/vegeta/lib"
)

func main() {
	types := map[string]interface{}{
		"Target": &vegeta.Target{},
	}

	valid := strings.Join(keys(types), ", ")

	fs := flag.NewFlagSet("jsonschema", flag.ContinueOnError)
	typ := fs.String("type", "", fmt.Sprintf("Vegeta type to generate a JSON schema for [%s]", valid))
	out := fs.String("output", "stdout", "Output file")

	if err := fs.Parse(os.Args[1:]); err != nil {
		die("%s", err)
	}

	t, ok := types[*typ]
	if !ok {
		die("invalid type %q not in [%s]", *typ, valid)
	}

	schema, err := json.MarshalIndent(jsonschema.Reflect(t), "", "  ")
	if err != nil {
		die("%s", err)
	}

	switch *out {
	case "stdout":
		_, err = os.Stdout.Write(schema)
	default:
		err = ioutil.WriteFile(*out, schema, 0644)
	}

	if err != nil {
		die("%s", err)
	}
}

func die(s string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, s, args...)
	os.Exit(1)
}

func keys(types map[string]interface{}) (ks []string) {
	for k := range types {
		ks = append(ks, k)
	}
	return ks
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helm

import (
	"bytes"
	"context"
	"fmt"
	"io"

	apimachinery "k8s.io/apimachinery/pkg/runtime/schema"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/helm"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/render"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/render/generate"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/render/renderer/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	sUtil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

type Helm struct {
	configName string
	generate.Generator
	config *latest.Helm

	kubeContext string
	kubeConfig  string
	namespace   string
	configFile  string
	labels      map[string]string
	enableDebug bool

	transformAllowlist map[apimachinery.GroupKind]latest.ResourceFilter
	transformDenylist  map[apimachinery.GroupKind]latest.ResourceFilter
}

func (h Helm) EnableDebug() bool         { return h.enableDebug }
func (h Helm) ConfigFile() string        { return h.configFile }
func (h Helm) KubeContext() string       { return h.kubeContext }
func (h Helm) KubeConfig() string        { return h.kubeConfig }
func (h Helm) Labels() map[string]string { return h.labels }
func (h Helm) GlobalFlags() []string     { return h.config.Flags.Global }

func New(cfg render.Config, rCfg latest.RenderConfig, labels map[string]string, configName string) (Helm, error) {
	generator := generate.NewGenerator(cfg.GetWorkingDir(), rCfg.Generate, "")
	transformAllowlist, transformDenylist, err := util.ConsolidateTransformConfiguration(cfg)
	if err != nil {
		return Helm{}, err
	}
	return Helm{
		configName: configName,
		Generator:  generator,
		config:     rCfg.Helm,

		enableDebug: cfg.Mode() == config.RunModes.Debug,
		configFile:  cfg.ConfigurationFile(),
		kubeContext: cfg.GetKubeContext(),
		kubeConfig:  cfg.GetKubeConfig(),
		labels:      labels,
		namespace:   cfg.GetNamespace(),

		transformAllowlist: transformAllowlist,
		transformDenylist:  transformDenylist,
	}, nil
}

func (h Helm) Render(ctx context.Context, out io.Writer, builds []graph.Artifact, _ bool) (manifest.ManifestListByConfig, error) {
	_, endTrace := instrumentation.StartTrace(ctx, "Render_HelmManifests")
	log.Entry(ctx).Infof("rendering using helm")
	instrumentation.AddAttributesToCurrentSpanFromContext(ctx, map[string]string{
		"RendererType": "helm",
	})

	manifests, err := h.generateHelmManifests(ctx, builds)
	endTrace()
	manifestListByConfig := manifest.NewManifestListByConfig()
	manifestListByConfig.Add(h.configName, manifests)
	return manifestListByConfig, err
}

func (h Helm) generateHelmManifests(ctx context.Context, builds []graph.Artifact) (manifest.ManifestList, error) {
	var renderedManifests manifest.ManifestList
	helmEnv := sUtil.OSEnviron()
	var postRendererArgs []string

	if len(builds) > 0 {
		skaffoldBinary, filterEnv, cleanup, err := helm.PrepareSkaffoldFilter(h, builds)
		if err != nil {
			return nil, fmt.Errorf("could not prepare `skaffold filter`: %w", err)
		}
		// need to include current environment, specifically for HOME to lookup ~/.kube/config
		helmEnv = append(helmEnv, filterEnv...)
		postRendererArgs = []string{"--post-renderer", skaffoldBinary}
		defer cleanup()
	}

	for _, release := range h.config.Releases {
		releaseName, err := sUtil.ExpandEnvTemplateOrFail(release.Name, nil)
		if err != nil {
			return nil, helm.UserErr(fmt.Sprintf("cannot expand release name %q", release.Name), err)
		}

		args := []string{"template", releaseName, helm.ChartSource(release)}
		args = append(args, postRendererArgs...)
		if release.Packaged == nil && release.Version != "" {
			args = append(args, "--version", release.Version)
		}

		args, err = helm.ConstructOverrideArgs(&release, builds, args)
		if err != nil {
			return nil, helm.UserErr("construct override args", err)
		}

		namespace, err := helm.ReleaseNamespace(h.namespace, release)
		if err != nil {
			return nil, err
		}
		if h.namespace != "" {
			namespace = h.namespace
		}
		if namespace != "" {
			args = append(args, "--namespace", namespace)
		}

		if release.Repo != "" {
			args = append(args, "--repo")
			args = append(args, release.Repo)
		}

		outBuffer := new(bytes.Buffer)
		errBuffer := new(bytes.Buffer)
		if err := helm.ExecWithStdoutAndStderr(ctx, h, outBuffer, errBuffer, false, helmEnv, args...); err != nil {
			return nil, helm.UserErr("std out err", fmt.Errorf(outBuffer.String(), fmt.Errorf(errBuffer.String())))
		}
		log.Entry(ctx).Errorf(errBuffer.String())
		renderedManifests.Append(outBuffer.Bytes())
	}

	manifests, err := renderedManifests.SetLabels(h.labels, manifest.NewResourceSelectorLabels(h.transformAllowlist, h.transformDenylist))
	if err != nil {
		return nil, err
	}

	return manifests, nil
}
<commit_msg>fix: not print error message if it is empty (#8005)<commit_after>/*
Copyright 2022 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helm

import (
	"bytes"
	"context"
	"fmt"
	"io"

	apimachinery "k8s.io/apimachinery/pkg/runtime/schema"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/helm"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/render"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/render/generate"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/render/renderer/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	sUtil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

type Helm struct {
	configName string
	generate.Generator
	config *latest.Helm

	kubeContext string
	kubeConfig  string
	namespace   string
	configFile  string
	labels      map[string]string
	enableDebug bool

	transformAllowlist map[apimachinery.GroupKind]latest.ResourceFilter
	transformDenylist  map[apimachinery.GroupKind]latest.ResourceFilter
}

func (h Helm) EnableDebug() bool         { return h.enableDebug }
func (h Helm) ConfigFile() string        { return h.configFile }
func (h Helm) KubeContext() string       { return h.kubeContext }
func (h Helm) KubeConfig() string        { return h.kubeConfig }
func (h Helm) Labels() map[string]string { return h.labels }
func (h Helm) GlobalFlags() []string     { return h.config.Flags.Global }

func New(cfg render.Config, rCfg latest.RenderConfig, labels map[string]string, configName string) (Helm, error) {
	generator := generate.NewGenerator(cfg.GetWorkingDir(), rCfg.Generate, "")
	transformAllowlist, transformDenylist, err := util.ConsolidateTransformConfiguration(cfg)
	if err != nil {
		return Helm{}, err
	}
	return Helm{
		configName: configName,
		Generator:  generator,
		config:     rCfg.Helm,

		enableDebug: cfg.Mode() == config.RunModes.Debug,
		configFile:  cfg.ConfigurationFile(),
		kubeContext: cfg.GetKubeContext(),
		kubeConfig:  cfg.GetKubeConfig(),
		labels:      labels,
		namespace:   cfg.GetNamespace(),

		transformAllowlist: transformAllowlist,
		transformDenylist:  transformDenylist,
	}, nil
}

func (h Helm) Render(ctx context.Context, out io.Writer, builds []graph.Artifact, _ bool) (manifest.ManifestListByConfig, error) {
	_, endTrace := instrumentation.StartTrace(ctx, "Render_HelmManifests")
	log.Entry(ctx).Infof("rendering using helm")
	instrumentation.AddAttributesToCurrentSpanFromContext(ctx, map[string]string{
		"RendererType": "helm",
	})

	manifests, err := h.generateHelmManifests(ctx, builds)
	endTrace()
	manifestListByConfig := manifest.NewManifestListByConfig()
	manifestListByConfig.Add(h.configName, manifests)
	return manifestListByConfig, err
}

func (h Helm) generateHelmManifests(ctx context.Context, builds []graph.Artifact) (manifest.ManifestList, error) {
	var renderedManifests manifest.ManifestList
	helmEnv := sUtil.OSEnviron()
	var postRendererArgs []string

	if len(builds) > 0 {
		skaffoldBinary, filterEnv, cleanup, err := helm.PrepareSkaffoldFilter(h, builds)
		if err != nil {
			return nil, fmt.Errorf("could not prepare `skaffold filter`: %w", err)
		}
		// need to include current environment, specifically for HOME to lookup ~/.kube/config
		helmEnv = append(helmEnv, filterEnv...)
		postRendererArgs = []string{"--post-renderer", skaffoldBinary}
		defer cleanup()
	}

	for _, release := range h.config.Releases {
		releaseName, err := sUtil.ExpandEnvTemplateOrFail(release.Name, nil)
		if err != nil {
			return nil, helm.UserErr(fmt.Sprintf("cannot expand release name %q", release.Name), err)
		}

		args := []string{"template", releaseName, helm.ChartSource(release)}
		args = append(args, postRendererArgs...)
		if release.Packaged == nil && release.Version != "" {
			args = append(args, "--version", release.Version)
		}

		args, err = helm.ConstructOverrideArgs(&release, builds, args)
		if err != nil {
			return nil, helm.UserErr("construct override args", err)
		}

		namespace, err := helm.ReleaseNamespace(h.namespace, release)
		if err != nil {
			return nil, err
		}
		if h.namespace != "" {
			namespace = h.namespace
		}
		if namespace != "" {
			args = append(args, "--namespace", namespace)
		}

		if release.Repo != "" {
			args = append(args, "--repo")
			args = append(args, release.Repo)
		}

		outBuffer := new(bytes.Buffer)
		errBuffer := new(bytes.Buffer)

		err = helm.ExecWithStdoutAndStderr(ctx, h, outBuffer, errBuffer, false, helmEnv, args...)
		errorMsg := errBuffer.String()

		if len(errorMsg) > 0 {
			log.Entry(ctx).Errorf(errorMsg)
		}

		if err != nil {
			return nil, helm.UserErr("std out err", fmt.Errorf(outBuffer.String(), fmt.Errorf(errorMsg)))
		}

		renderedManifests.Append(outBuffer.Bytes())
	}

	manifests, err := renderedManifests.SetLabels(h.labels, manifest.NewResourceSelectorLabels(h.transformAllowlist, h.transformDenylist))
	if err != nil {
		return nil, err
	}

	return manifests, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"flag"
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/jarosser06/fastfood"
)

const (
	tempPackEnvVar = "FASTFOOD_TEMPLATE_PACK"
)

type Generator struct {
	MappedArgs    map[string]string
	TemplatesPath string
}

// Translates key:value strings into a map
func MapArgs(args []string) map[string]string {
	var argMap map[string]string
	argMap = make(map[string]string)

	for _, arg := range args {
		if strings.Contains(arg, ":") {
			// Split at the first : in an arg
			splitArg := strings.SplitN(arg, ":", 2)

			argMap[splitArg[0]] = splitArg[1]
		}
	}

	return argMap
}

func DefTempPack() string {
	packEnv := os.Getenv("FASTFOOD_TEMPLATE_PACK")
	if packEnv == "" {
		return path.Join(os.Getenv("HOME"), "fastfood")
	} else {
		return packEnv
	}
}

func (g *Generator) Run(args []string) int {
	workingDir, _ := os.Getwd()
	cmdFlags := flag.NewFlagSet("gen", flag.ContinueOnError)
	cmdFlags.Usage = func() { fmt.Println(g.Help()) }

	templatePack := *cmdFlags.String("templates-pack", DefTempPack(), "path to the templates directory")

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if fastfood.PathIsCookbook(workingDir) {
		ckbk, err := fastfood.NewCookbookFromPath(workingDir)

		if err != nil {
			fmt.Println("Unable to parse cookbook")
			return 1
		}

		cmdManifest := path.Join(templatePack, "manifest.json")
		if !fastfood.FileExist(cmdManifest) {
			fmt.Printf("Error no such file %s\n", cmdManifest)
			return 1
		}

		// Remove the first arg as the command
		genCommand, args := args[0], args[1:len(args)]

		commands := ParseCommandsFromFile(cmdManifest)
		if _, ok := commands[genCommand]; ok {
			goto CMDFound
		}

		// If the loop finishes without finding the commnad exit
		fmt.Printf("No generator found for %s\n", genCommand)
		return 1

		// Command was found continue to execute
	CMDFound:

		p, err := fastfood.NewProviderFromFile(
			ckbk,
			path.Join(templatePack, commands[genCommand].Manifest),
		)

		if err != nil {
			fmt.Println("Error loading provider %s: %v", genCommand, err)
			return 1
		}

		mappedArgs := MapArgs(args)
		var providerType string
		if val, ok := mappedArgs["type"]; ok {
			providerType = val
		} else {
			if p.DefaultType != "" {
				providerType = p.DefaultType
			} else {
				fmt.Println("You must pass a type b/c not default type is set")
				return 1
			}
		}

		p.GenDirs(providerType)
		err = p.GenFiles(
			providerType,
			path.Join(templatePack, genCommand),
			mappedArgs,
		)

		if err != nil {
			fmt.Printf("Error generating files %v\n", err)
			return 1
		}

	} else {
		fmt.Println("You must run this command from a cookbook directory")
		return 1
	}
	return 0
}

func (g *Generator) Synopsis() string {
	return "Generates a new recipe for an existing cookbook"
}

// Autogenerate based on commands parsed
func (g *Generator) Help() string {
	helpText := `
Usage: fastfood gen [provider] [options]

  This will generate a recipe and spec file
  based on the provider and options you
  provide that provider.

  Options are passed using using a key:value
  notation so to set the name you would use
  the following:

  name:recipe_name

Generators:

  db     - Creates a database recipe based
           on the type, defaults to MySQL

  app    - Creates an application recipe
           based on the type, defaults to Generic`

	return helpText
}
<commit_msg>re-enabled metatdata appending<commit_after>package cmd

import (
	"flag"
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/jarosser06/fastfood"
)

const (
	tempPackEnvVar = "FASTFOOD_TEMPLATE_PACK"
)

type Generator struct {
	MappedArgs    map[string]string
	TemplatesPath string
}

// Translates key:value strings into a map
func MapArgs(args []string) map[string]string {
	var argMap map[string]string
	argMap = make(map[string]string)

	for _, arg := range args {
		if strings.Contains(arg, ":") {
			// Split at the first : in an arg
			splitArg := strings.SplitN(arg, ":", 2)

			argMap[splitArg[0]] = splitArg[1]
		}
	}

	return argMap
}

func DefTempPack() string {
	packEnv := os.Getenv("FASTFOOD_TEMPLATE_PACK")
	if packEnv == "" {
		return path.Join(os.Getenv("HOME"), "fastfood")
	} else {
		return packEnv
	}
}

func (g *Generator) Run(args []string) int {
	workingDir, _ := os.Getwd()
	cmdFlags := flag.NewFlagSet("gen", flag.ContinueOnError)
	cmdFlags.Usage = func() { fmt.Println(g.Help()) }

	templatePack := *cmdFlags.String("templates-pack", DefTempPack(), "path to the templates directory")

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if fastfood.PathIsCookbook(workingDir) {
		ckbk, err := fastfood.NewCookbookFromPath(workingDir)

		if err != nil {
			fmt.Println("Unable to parse cookbook")
			return 1
		}

		cmdManifest := path.Join(templatePack, "manifest.json")
		if !fastfood.FileExist(cmdManifest) {
			fmt.Printf("Error no such file %s\n", cmdManifest)
			return 1
		}

		// Remove the first arg as the command
		genCommand, args := args[0], args[1:len(args)]

		commands := ParseCommandsFromFile(cmdManifest)
		if _, ok := commands[genCommand]; ok {
			goto CMDFound
		}

		// If the loop finishes without finding the commnad exit
		fmt.Printf("No generator found for %s\n", genCommand)
		return 1

		// Command was found continue to execute
	CMDFound:

		p, err := fastfood.NewProviderFromFile(
			ckbk,
			path.Join(templatePack, commands[genCommand].Manifest),
		)

		if err != nil {
			fmt.Println("Error loading provider %s: %v", genCommand, err)
			return 1
		}

		mappedArgs := MapArgs(args)
		var providerType string
		if val, ok := mappedArgs["type"]; ok {
			providerType = val
		} else {
			if p.DefaultType != "" {
				providerType = p.DefaultType
			} else {
				fmt.Println("You must pass a type b/c not default type is set")
				return 1
			}
		}

		// Add the needed dependencies to the metadata
		ckbk.AppendDependencies(p.Dependencies(providerType))
		p.GenDirs(providerType)

		err = p.GenFiles(
			providerType,
			path.Join(templatePack, genCommand),
			mappedArgs,
		)

		if err != nil {
			fmt.Printf("Error generating files %v\n", err)
			return 1
		}

	} else {
		fmt.Println("You must run this command from a cookbook directory")
		return 1
	}
	return 0
}

func (g *Generator) Synopsis() string {
	return "Generates a new recipe for an existing cookbook"
}

// Autogenerate based on commands parsed
func (g *Generator) Help() string {
	helpText := `
Usage: fastfood gen [provider] [options]

  This will generate a recipe and spec file
  based on the provider and options you
  provide that provider.

  Options are passed using using a key:value
  notation so to set the name you would use
  the following:

  name:recipe_name

Generators:

  db     - Creates a database recipe based
           on the type, defaults to MySQL

  app    - Creates an application recipe
           based on the type, defaults to Generic`

	return helpText
}
<|endoftext|>


<commit_before>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"bytes"
	"errors"
)

const (
	FrameHeaderSize = 10
)

// FrameType holds frame id metadata and constructor method
// A set number of these are created in the version specific files
type FrameType struct {
	id          string
	description string
	constructor func(FrameHead, []byte) Framer
}

// Framer provides a generic interface for frames
// This is the default type returned when creating frames
type Framer interface {
	Id() string
	Size() int
	StatusFlags() byte
	FormatFlags() byte
	String() string
	Bytes() []byte
}

// FrameHead represents the header of each frame
// Additional metadata is kept through the embedded frame type
// These do not usually need to be manually created
type FrameHead struct {
	FrameType
	statusFlags byte
	formatFlags byte
	size        int32
}

func (h FrameHead) Id() string {
	return h.id
}

func (h FrameHead) Size() int {
	return int(h.size)
}

func (h FrameHead) StatusFlags() byte {
	return h.statusFlags
}

func (h FrameHead) FormatFlags() byte {
	return h.formatFlags
}

// DataFrame is the default frame for binary data
type DataFrame struct {
	FrameHead
	data []byte
}

func NewDataFrame(head FrameHead, data []byte) Framer {
	return &DataFrame{head, data}
}

func (f DataFrame) Data() []byte {
	return f.data
}

func (f *DataFrame) SetData(b []byte) {
	f.size += int32(len(b)) - f.size
	f.data = b
}

func (f DataFrame) String() string {
	return "<binary data>"
}

func (f DataFrame) Bytes() []byte {
	return f.data
}

// TextFramer represents frames that contain encoded text
type TextFramer interface {
	Framer
	Encoding() string
	SetEncoding(string) error
	Text() string
	SetText(string) error
}

// TextFrame represents frames that contain encoded text
type TextFrame struct {
	FrameHead
	encoding byte
	text     string
}

func NewTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &TextFrame{FrameHead: head}

	f.encoding = data[0]

	if f.text, err = Decoders[f.encoding].ConvertString(string(data[1:])); err != nil {
		return nil
	}

	return f
}

func (f TextFrame) Encoding() string {
	return encodingForIndex(f.encoding)
}

func (f *TextFrame) SetEncoding(encoding string) error {
	i := byte(indexForEncoding(encoding))
	if i < 0 {
		return errors.New("encoding: invalid encoding")
	}

	f.encoding = i
	return nil
}

func (f TextFrame) Text() string {
	return f.text
}

func (f *TextFrame) SetText(text string) error {
	diff, err := encodedDiff(f.encoding, text, f.text)
	if err != nil {
		return err
	}

	f.size += int32(diff)
	f.text = text
	return nil
}

func (f TextFrame) String() string {
	return f.text
}

func (f TextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedString, err := Encoders[f.encoding].ConvertString(f.text)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	copy(bytes[1:], []byte(encodedString))

	return bytes
}

type DescTextFrame struct {
	FrameHead
	TextFrame
	description string
}

// DescTextFrame represents frames that contain encoded text and descriptions
func NewDescTextFrame(head FrameHead, data []byte) Framer {
	f := &DescTextFrame{FrameHead: head}

	var err error

	f.encoding = data[0]

	cutoff := 1
	if i := afterNullIndex(data[1:], f.encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.description, err = Decoders[f.encoding].ConvertString(string(data[1:cutoff])); err != nil {
		return nil
	}

	if f.text, err = Decoders[f.encoding].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

func (f DescTextFrame) Description() string {
	return f.description
}

func (f *DescTextFrame) SetDescription(description string) error {
	diff, err := encodedDiff(f.encoding, description, f.description)
	if err != nil {
		return err
	}

	f.size += int32(diff)
	f.description = description
	return nil
}

func (f DescTextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedDescription, err := Encoders[f.encoding].ConvertString(f.description)
	if err != nil {
		return bytes
	}
	encodedText, err := Encoders[f.encoding].ConvertString(f.text)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	index := 1
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(encodedText)], []byte(encodedText))

	return bytes
}

// UnsynchTextFrame represents frames that contain unsynchronized text
type UnsynchTextFrame struct {
	FrameHead
	DescTextFrame
	language string
}

func NewUnsynchTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &UnsynchTextFrame{FrameHead: head}

	f.encoding = data[0]
	f.language = string(data[1:4])

	cutoff := 4
	if i := afterNullIndex(data[4:], f.encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.description, err = Decoders[f.encoding].ConvertString(string(data[4:cutoff])); err != nil {
		return nil
	}

	if f.text, err = Decoders[f.encoding].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

func (f UnsynchTextFrame) Language() string {
	return f.language
}

func (f *UnsynchTextFrame) SetLanguage(language string) error {
	if len(language) != 3 {
		return errors.New("language: invalid language string")
	}

	f.language = language
	return nil
}

func (f UnsynchTextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedDescription, err := Encoders[f.encoding].ConvertString(f.description)
	if err != nil {
		return bytes
	}
	encodedText, err := Encoders[f.encoding].ConvertString(f.text)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	copy(bytes[1:4], []byte(f.language))
	index := 4
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(encodedText)], []byte(encodedText))

	return bytes
}

// ImageFrame represent frames that have media attached
type ImageFrame struct {
	FrameHead
	DataFrame
	encoding    byte
	mimeType    string
	pictureType byte
	description string
}

func NewImageFrame(head FrameHead, data []byte) Framer {
	f := &ImageFrame{FrameHead: head}

	var err error
	encodingIndex := data[0]

	f.encoding = encodingIndex

	buffer := bytes.NewBuffer(data[1:])
	if f.mimeType, err = buffer.ReadString(0); err != nil {
		return nil
	}

	if f.pictureType, err = buffer.ReadByte(); err != nil {
		return nil
	}

	beginIndex := 1 + len(f.mimeType) + 1
	var cutoff int
	if i := afterNullIndex(data[beginIndex:], f.encoding); i < 0 {
		return nil
	} else {
		cutoff = beginIndex + i
	}

	if f.description, err = Decoders[encodingIndex].ConvertString(string(data[beginIndex:cutoff])); err != nil {
		return nil
	}

	f.data = data[cutoff:]

	return f
}

func (f ImageFrame) Encoding() string {
	return encodingForIndex(f.encoding)
}

func (f *ImageFrame) SetEncoding(encoding string) error {
	i := byte(indexForEncoding(encoding))
	if i < 0 {
		return errors.New("encoding: invalid encoding")
	}

	f.encoding = i
	return nil
}

func (f ImageFrame) MIMEType() string {
	return f.mimeType
}

func (f *ImageFrame) SetMIMEType(mimeType string) {
	f.size += int32(len(mimeType)) - f.size
	if mimeType[len(mimeType)-1] != 0 {
		nullTermBytes := append([]byte(mimeType), 0x00)
		f.mimeType = string(nullTermBytes)
		f.size += 1
	} else {
		f.mimeType = mimeType
	}
}

func (f ImageFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedDescription, err := Encoders[f.encoding].ConvertString(f.description)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	index := 1
	copy(bytes[index:index+len(f.mimeType)], []byte(f.mimeType))
	index += len(f.mimeType)
	bytes[index] = f.pictureType
	index += 1
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(f.data)], f.data)

	return bytes
}
<commit_msg>Change Framer String() methods<commit_after>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"bytes"
	"errors"
)

const (
	FrameHeaderSize = 10
)

// FrameType holds frame id metadata and constructor method
// A set number of these are created in the version specific files
type FrameType struct {
	id          string
	description string
	constructor func(FrameHead, []byte) Framer
}

// Framer provides a generic interface for frames
// This is the default type returned when creating frames
type Framer interface {
	Id() string
	Size() int
	StatusFlags() byte
	FormatFlags() byte
	String() string
	Bytes() []byte
}

// FrameHead represents the header of each frame
// Additional metadata is kept through the embedded frame type
// These do not usually need to be manually created
type FrameHead struct {
	FrameType
	statusFlags byte
	formatFlags byte
	size        int32
}

func (h FrameHead) Id() string {
	return h.id
}

func (h FrameHead) Size() int {
	return int(h.size)
}

func (h FrameHead) StatusFlags() byte {
	return h.statusFlags
}

func (h FrameHead) FormatFlags() byte {
	return h.formatFlags
}

// DataFrame is the default frame for binary data
type DataFrame struct {
	FrameHead
	data []byte
}

func NewDataFrame(head FrameHead, data []byte) Framer {
	return &DataFrame{head, data}
}

func (f DataFrame) Data() []byte {
	return f.data
}

func (f *DataFrame) SetData(b []byte) {
	f.size += int32(len(b)) - f.size
	f.data = b
}

func (f DataFrame) String() string {
	return "<binary data>"
}

func (f DataFrame) Bytes() []byte {
	return f.data
}

// TextFramer represents frames that contain encoded text
type TextFramer interface {
	Framer
	Encoding() string
	SetEncoding(string) error
	Text() string
	SetText(string) error
}

// TextFrame represents frames that contain encoded text
type TextFrame struct {
	FrameHead
	encoding byte
	text     string
}

func NewTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &TextFrame{FrameHead: head}

	f.encoding = data[0]

	if f.text, err = Decoders[f.encoding].ConvertString(string(data[1:])); err != nil {
		return nil
	}

	return f
}

func (f TextFrame) Encoding() string {
	return encodingForIndex(f.encoding)
}

func (f *TextFrame) SetEncoding(encoding string) error {
	i := byte(indexForEncoding(encoding))
	if i < 0 {
		return errors.New("encoding: invalid encoding")
	}

	f.encoding = i
	return nil
}

func (f TextFrame) Text() string {
	return f.text
}

func (f *TextFrame) SetText(text string) error {
	diff, err := encodedDiff(f.encoding, text, f.text)
	if err != nil {
		return err
	}

	f.size += int32(diff)
	f.text = text
	return nil
}

func (f TextFrame) String() string {
	return f.text
}

func (f TextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedString, err := Encoders[f.encoding].ConvertString(f.text)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	copy(bytes[1:], []byte(encodedString))

	return bytes
}

type DescTextFrame struct {
	FrameHead
	TextFrame
	description string
}

// DescTextFrame represents frames that contain encoded text and descriptions
func NewDescTextFrame(head FrameHead, data []byte) Framer {
	f := &DescTextFrame{FrameHead: head}

	var err error

	f.encoding = data[0]

	cutoff := 1
	if i := afterNullIndex(data[1:], f.encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.description, err = Decoders[f.encoding].ConvertString(string(data[1:cutoff])); err != nil {
		return nil
	}

	if f.text, err = Decoders[f.encoding].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

func (f DescTextFrame) Description() string {
	return f.description
}

func (f *DescTextFrame) SetDescription(description string) error {
	diff, err := encodedDiff(f.encoding, description, f.description)
	if err != nil {
		return err
	}

	f.size += int32(diff)
	f.description = description
	return nil
}

func (f DescTextFrame) String() string {
	return fmt.Sprintf("%s: %s", f.description, f.text)
}

func (f DescTextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedDescription, err := Encoders[f.encoding].ConvertString(f.description)
	if err != nil {
		return bytes
	}
	encodedText, err := Encoders[f.encoding].ConvertString(f.text)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	index := 1
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(encodedText)], []byte(encodedText))

	return bytes
}

// UnsynchTextFrame represents frames that contain unsynchronized text
type UnsynchTextFrame struct {
	FrameHead
	DescTextFrame
	language string
}

func NewUnsynchTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &UnsynchTextFrame{FrameHead: head}

	f.encoding = data[0]
	f.language = string(data[1:4])

	cutoff := 4
	if i := afterNullIndex(data[4:], f.encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.description, err = Decoders[f.encoding].ConvertString(string(data[4:cutoff])); err != nil {
		return nil
	}

	if f.text, err = Decoders[f.encoding].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

func (f UnsynchTextFrame) Language() string {
	return f.language
}

func (f *UnsynchTextFrame) SetLanguage(language string) error {
	if len(language) != 3 {
		return errors.New("language: invalid language string")
	}

	f.language = language
	return nil
}

func (f UnsynchTextFrame) String() string {
	return fmt.Sprintf("%s\t%s:\n%s", f.language, f.description, f.text)
}

func (f UnsynchTextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedDescription, err := Encoders[f.encoding].ConvertString(f.description)
	if err != nil {
		return bytes
	}
	encodedText, err := Encoders[f.encoding].ConvertString(f.text)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	copy(bytes[1:4], []byte(f.language))
	index := 4
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(encodedText)], []byte(encodedText))

	return bytes
}

// ImageFrame represent frames that have media attached
type ImageFrame struct {
	FrameHead
	DataFrame
	encoding    byte
	mimeType    string
	pictureType byte
	description string
}

func NewImageFrame(head FrameHead, data []byte) Framer {
	f := &ImageFrame{FrameHead: head}

	var err error
	encodingIndex := data[0]

	f.encoding = encodingIndex

	buffer := bytes.NewBuffer(data[1:])
	if f.mimeType, err = buffer.ReadString(0); err != nil {
		return nil
	}

	if f.pictureType, err = buffer.ReadByte(); err != nil {
		return nil
	}

	beginIndex := 1 + len(f.mimeType) + 1
	var cutoff int
	if i := afterNullIndex(data[beginIndex:], f.encoding); i < 0 {
		return nil
	} else {
		cutoff = beginIndex + i
	}

	if f.description, err = Decoders[encodingIndex].ConvertString(string(data[beginIndex:cutoff])); err != nil {
		return nil
	}

	f.data = data[cutoff:]

	return f
}

func (f ImageFrame) Encoding() string {
	return encodingForIndex(f.encoding)
}

func (f *ImageFrame) SetEncoding(encoding string) error {
	i := byte(indexForEncoding(encoding))
	if i < 0 {
		return errors.New("encoding: invalid encoding")
	}

	f.encoding = i
	return nil
}

func (f ImageFrame) MIMEType() string {
	return f.mimeType
}

func (f *ImageFrame) SetMIMEType(mimeType string) {
	f.size += int32(len(mimeType)) - f.size
	if mimeType[len(mimeType)-1] != 0 {
		nullTermBytes := append([]byte(mimeType), 0x00)
		f.mimeType = string(nullTermBytes)
		f.size += 1
	} else {
		f.mimeType = mimeType
	}
}

func (f ImageFrame) String() string {
	return fmt.Sprintf("%s\t%s: <binary data>", f.mimeType, f.description)
}

func (f ImageFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodedDescription, err := Encoders[f.encoding].ConvertString(f.description)
	if err != nil {
		return bytes
	}

	bytes[0] = f.encoding
	index := 1
	copy(bytes[index:index+len(f.mimeType)], []byte(f.mimeType))
	index += len(f.mimeType)
	bytes[index] = f.pictureType
	index += 1
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(f.data)], f.data)

	return bytes
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"net/url"
	"testing"
)

func TestCtlV3EndpointHealth(t *testing.T) { testCtl(t, endpointHealthTest, withQuorum()) }
func TestCtlV3EndpointStatus(t *testing.T) { testCtl(t, endpointStatusTest, withQuorum()) }
func TestCtlV3EndpointHealthWithAuth(t *testing.T) {
	testCtl(t, endpointHealthTestWithAuth, withQuorum())
}

func endpointHealthTest(cx ctlCtx) {
	if err := ctlV3EndpointHealth(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}
}

func ctlV3EndpointHealth(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "endpoint", "health")
	lines := make([]string, cx.epc.cfg.clusterSize)
	for i := range lines {
		lines[i] = "is healthy"
	}
	return spawnWithExpects(cmdArgs, lines...)
}

func ctlV3EndpointHealthWithKey(cx ctlCtx, key string) error {
	cmdArgs := append(cx.PrefixArgs(), "endpoint", "health", "--health-check-key", key)
	lines := make([]string, cx.epc.cfg.clusterSize)
	for i := range lines {
		lines[i] = "is healthy"
	}
	return spawnWithExpects(cmdArgs, lines...)
}

func endpointStatusTest(cx ctlCtx) {
	if err := ctlV3EndpointStatus(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointStatus error (%v)", err)
	}
}

func ctlV3EndpointStatus(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "endpoint", "status")
	var eps []string
	for _, ep := range cx.epc.endpoints() {
		u, _ := url.Parse(ep)
		eps = append(eps, u.Host)
	}
	return spawnWithExpects(cmdArgs, eps...)
}

func ctlV3EndpointHealthFailPermissionDenied(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "endpoint", "health")
	lines := make([]string, cx.epc.cfg.clusterSize)
	for i := range lines {
		lines[i] = "is unhealthy: failed to commit proposal: etcdserver: permission denied"
	}
	return spawnWithExpects(cmdArgs, lines...)
}

func endpointHealthTestWithAuth(cx ctlCtx) {
	if err := authEnable(cx); err != nil {
		cx.t.Fatal(err)
	}

	cx.user, cx.pass = "root", "root"
	authSetupTestUser(cx)

	if err := ctlV3EndpointHealth(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}

	// health checking with an ordinal user must fail because the user isn't granted a permission of the key "health"
	cx.user, cx.pass = "test-user", "pass"
	if err := ctlV3EndpointHealthFailPermissionDenied(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}

	cx.user, cx.pass = "root", "root"
	if err := ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "custom-key", "", false}); err != nil {
		cx.t.Fatal(err)
	}

	cx.user, cx.pass = "test-user", "pass"
	if err := ctlV3EndpointHealthWithKey(cx, "custom-key"); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}
}
<commit_msg>e2e: check etcdctl endpoint health is healthy if denied permission to key<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"net/url"
	"testing"
)

func TestCtlV3EndpointHealth(t *testing.T) { testCtl(t, endpointHealthTest, withQuorum()) }
func TestCtlV3EndpointStatus(t *testing.T) { testCtl(t, endpointStatusTest, withQuorum()) }
func TestCtlV3EndpointHealthWithAuth(t *testing.T) {
	testCtl(t, endpointHealthTestWithAuth, withQuorum())
}

func endpointHealthTest(cx ctlCtx) {
	if err := ctlV3EndpointHealth(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}
}

func ctlV3EndpointHealth(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "endpoint", "health")
	lines := make([]string, cx.epc.cfg.clusterSize)
	for i := range lines {
		lines[i] = "is healthy"
	}
	return spawnWithExpects(cmdArgs, lines...)
}

func endpointStatusTest(cx ctlCtx) {
	if err := ctlV3EndpointStatus(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointStatus error (%v)", err)
	}
}

func ctlV3EndpointStatus(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "endpoint", "status")
	var eps []string
	for _, ep := range cx.epc.endpoints() {
		u, _ := url.Parse(ep)
		eps = append(eps, u.Host)
	}
	return spawnWithExpects(cmdArgs, eps...)
}

func endpointHealthTestWithAuth(cx ctlCtx) {
	if err := authEnable(cx); err != nil {
		cx.t.Fatal(err)
	}

	cx.user, cx.pass = "root", "root"
	authSetupTestUser(cx)

	if err := ctlV3EndpointHealth(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}

	// health checking with an ordinal user "succeeds" since permission denial goes through consensus
	cx.user, cx.pass = "test-user", "pass"
	if err := ctlV3EndpointHealth(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}

	// succeed if permissions granted for ordinary user
	cx.user, cx.pass = "root", "root"
	if err := ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "health", "", false}); err != nil {
		cx.t.Fatal(err)
	}
	cx.user, cx.pass = "test-user", "pass"
	if err := ctlV3EndpointHealth(cx); err != nil {
		cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err)
	}
}
<|endoftext|>


<commit_before>// Package crawler implements asynchronous web sites crawler using interfaces
// from tasker and ability to specify number of workers and size of tasks' buffer.
package crawler

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/krasoffski/gomill/tasker"
)

var result = map[bool]string{
	true:  color.GreenString("PASS"),
	false: color.RedString("FAIL"),
}

// httpTask represents HTTP task with required for processing fields.
// ok idecates that url is fetched without issues within timeout
// url is URL to fetch
// bsize is body size of response
// start is start time of processing
// client is http.Client with timeout
// elapsed is elapsed time for processing
type httpTask struct {
	ok      bool
	url     string
	bsize   int64
	start   time.Time
	client  *http.Client
	elapsed time.Duration
}

// Process processes and fills required fields of HTTPTask.
func (h *httpTask) Process() {
	h.start = time.Now()
	resp, err := h.client.Get(h.url)
	h.elapsed = time.Since(h.start)

	if err != nil {
		h.ok = false
		return
	}
	defer resp.Body.Close()

	// TODO: Think about handling error for Copy.
	h.bsize, _ = io.Copy(ioutil.Discard, resp.Body)

	if resp.StatusCode == http.StatusOK {
		h.ok = true
		return
	}
	h.ok = false
}

// Output prints out Task result to standart output.
func (h *httpTask) Output() {
	secs := color.BlueString("[%7.2fs]", h.elapsed.Seconds())
	bsize := color.YellowString("[%10db]", h.bsize)
	fmt.Printf("%s %s %s %s\n", result[h.ok], secs, bsize, h.url)
}

type taskBuilder struct {
	source     io.Reader
	bufSize    int
	httpClient *http.Client
}

func (tb *taskBuilder) BufSize() int {
	return tb.bufSize
}

func (tb *taskBuilder) Create(url string) tasker.Task {
	h := new(httpTask)
	h.url = url
	h.client = tb.httpClient
	return h
}

func (tb *taskBuilder) Items() <-chan string {
	urls := make(chan string, tb.bufSize)
	s := bufio.NewScanner(tb.source)
	go func() {
		defer close(urls)
		for s.Scan() {

			line := strings.TrimSpace(s.Text())

			if line == "" || strings.HasPrefix(line, "#") {
				continue
			}
			urls <- line
		}
	}()
	if s.Err() != nil {
		log.Fatalf("error reading: %s", s.Err())
	}
	return urls
}

// Run creates tasks and process them with given number of workers.
func (tb *taskBuilder) Run(workers int) {
	tasker.Run(tb, workers)
}

// New creates and initializes new task builder.
func New(r io.Reader, bufSize int, client *http.Client) tasker.Builder {
	m := new(taskBuilder)
	m.source = r
	m.bufSize = bufSize
	m.httpClient = client
	return m
}
<commit_msg>updated docstring for crawler<commit_after>// Package crawler implements asynchronous web sites crawler using interfaces
// from tasker and ability to specify number of workers and size of tasks' buffer.
package crawler

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/krasoffski/gomill/tasker"
)

var result = map[bool]string{
	true:  color.GreenString("PASS"),
	false: color.RedString("FAIL"),
}

// httpTask represents HTTP task with required for processing fields.
// ok idecates that url is fetched without issues within timeout
// url is URL to fetch
// bsize is body size of response
// start is start time of processing
// client is http.Client with timeout
// elapsed is elapsed time for processing
type httpTask struct {
	ok      bool
	url     string
	bsize   int64
	start   time.Time
	client  *http.Client
	elapsed time.Duration
}

// Process processes and fills required fields of HTTPTask.
func (h *httpTask) Process() {
	h.start = time.Now()
	resp, err := h.client.Get(h.url)
	h.elapsed = time.Since(h.start)

	if err != nil {
		h.ok = false
		return
	}
	defer resp.Body.Close()

	// TODO: Think about handling error for Copy.
	h.bsize, _ = io.Copy(ioutil.Discard, resp.Body)

	if resp.StatusCode == http.StatusOK {
		h.ok = true
		return
	}
	h.ok = false
}

// Output prints out Task result to standart output.
func (h *httpTask) Output() {
	secs := color.BlueString("[%7.2fs]", h.elapsed.Seconds())
	bsize := color.YellowString("[%10db]", h.bsize)
	fmt.Printf("%s %s %s %s\n", result[h.ok], secs, bsize, h.url)
}

type taskBuilder struct {
	source     io.Reader
	bufSize    int
	httpClient *http.Client
}

func (tb *taskBuilder) BufSize() int {
	return tb.bufSize
}

func (tb *taskBuilder) Create(url string) tasker.Task {
	h := new(httpTask)
	h.url = url
	h.client = tb.httpClient
	return h
}

func (tb *taskBuilder) Items() <-chan string {
	urls := make(chan string, tb.bufSize)
	s := bufio.NewScanner(tb.source)
	go func() {
		defer close(urls)
		for s.Scan() {

			line := strings.TrimSpace(s.Text())

			if line == "" || strings.HasPrefix(line, "#") {
				continue
			}
			urls <- line
		}
	}()
	if s.Err() != nil {
		log.Fatalf("error reading: %s", s.Err())
	}
	return urls
}

// Run processes the tasks which are created internally using Create method.
// It blocks execution and waits till all tasks completed.
func (tb *taskBuilder) Run(workers int) {
	tasker.Run(tb, workers)
}

// New creates and initializes new task builder.
func New(r io.Reader, bufSize int, client *http.Client) tasker.Builder {
	m := new(taskBuilder)
	m.source = r
	m.bufSize = bufSize
	m.httpClient = client
	return m
}
<|endoftext|>


<commit_before>package main

import (
	"image"
	"image/draw"
	"sync"

	"github.com/as/frame"
	"golang.org/x/exp/shiny/driver"
	"golang.org/x/exp/shiny/screen"
	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/lifecycle"
	"golang.org/x/mobile/event/mouse"
	"golang.org/x/mobile/event/paint"
	"golang.org/x/mobile/event/size"
)

var wg sync.WaitGroup
var winSize = image.Pt(1024, 768)

func pt(e mouse.Event) image.Point {
	return image.Pt(int(e.X), int(e.Y))
}

func main() {
	driver.Main(func(src screen.Screen) {
		var dirty = true
		wind, _ := src.NewWindow(&screen.NewWindowOptions{winSize.X, winSize.Y, "basic"})
		b, _ := src.NewBuffer(winSize)
		draw.Draw(b.RGBA(), b.Bounds(), frame.A.Back, image.ZP, draw.Src)
		fr := frame.New(b.RGBA(), b.Bounds(), nil)
		fr.Refresh()
		wind.Send(paint.Event{})
		ck := func() {
			if dirty || fr.Dirty() {
				wind.Send(paint.Event{})
			}
			dirty = false
		}

		for {
			switch e := wind.NextEvent().(type) {
			case mouse.Event:
				if e.Button == 1 && e.Direction == 1 {
					p0 := fr.IndexOf(pt(e))
					fr.Select(p0, p0)
					flush := func() {
						wind.Upload(fr.Bounds().Min, b, fr.Bounds())
						wind.Publish()
					}
					flush()
					fr.Sweep(wind, flush)
					wind.Send(paint.Event{})
				}
			case key.Event:
				if e.Direction == 2 {
					continue
				}
				if e.Rune == '\r' {
					e.Rune = '\n'
				}
				if e.Rune > 0x79 || e.Rune < 0 {
					continue
				}
				p0, p1 := fr.Dot()
				if e.Rune == '\x08' {
					if p0 == p1 && p0 > 0 {
						p0--
					}
					fr.Delete(p0, p1)
				} else {
					fr.Insert([]byte{byte(e.Rune)}, p0)
					p0++
				}
				fr.Select(p0, p0)
				dirty = true
				ck()
			case size.Event:
				wind.Upload(image.ZP, b, b.Bounds())
				fr.Refresh()
				ck()
			case paint.Event:
				wind.Upload(fr.Bounds().Min, b, fr.Bounds())
				fr.Flush()
				wind.Publish()
			case lifecycle.Event:
				if e.To == lifecycle.StageDead {
					return
				}
			}
		}
	})
}
<commit_msg>frame: update basic example<commit_after>package main

import (
	"image"
	"image/draw"
	"sync"

	"github.com/as/frame"
	"golang.org/x/exp/shiny/driver"
	"golang.org/x/exp/shiny/screen"
	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/lifecycle"
	"golang.org/x/mobile/event/mouse"
	"golang.org/x/mobile/event/paint"
	"golang.org/x/mobile/event/size"
)

var wg sync.WaitGroup
var winSize = image.Pt(1024, 768)

func pt(e mouse.Event) image.Point {
	return image.Pt(int(e.X), int(e.Y))
}

func main() {
	driver.Main(func(src screen.Screen) {
		var dirty = true
		wind, _ := src.NewWindow(&screen.NewWindowOptions{winSize.X, winSize.Y, "basic"})
		b, _ := src.NewBuffer(winSize)
		draw.Draw(b.RGBA(), b.Bounds(), frame.A.Back, image.ZP, draw.Src)
		fr := frame.New(b.RGBA(), b.Bounds(), nil)
		fr.Refresh()
		wind.Send(paint.Event{})
		ck := func() {
			if dirty || fr.Dirty() {
				wind.Send(paint.Event{})
			}
			dirty = false
		}

		for {
			switch e := wind.NextEvent().(type) {
			case mouse.Event:
				if e.Button == 1 && e.Direction == 1 {
					p0 := fr.IndexOf(pt(e))
					fr.Select(p0, p0)
					flush := func() {
						wind.Upload(fr.Bounds().Min, b, fr.Bounds())
						wind.Publish()
					}
					flush()
					fr.Sweep(wind, flush)
					wind.Send(paint.Event{})
				}
			case key.Event:
				if e.Direction == 2 {
					continue
				}
				if e.Rune == '\r' {
					e.Rune = '\n'
				}
				p0, p1 := fr.Dot()
				if e.Rune > 0x79 || e.Rune < 0 {
					continue
				}
				if e.Rune == '\x08' {
					if p0 == p1 && p0 > 0 {
						p0--
					}
					fr.Delete(p0, p1)
				} else {
					if p0 != p1 {
						fr.Delete(p0, p1)
					}
					fr.Insert([]byte{byte(e.Rune)}, p0)
				}
				dirty = true
				ck()
			case size.Event:
				wind.Upload(image.ZP, b, b.Bounds())
				fr.Refresh()
				ck()
			case paint.Event:
				wind.Upload(fr.Bounds().Min, b, fr.Bounds())
				fr.Flush()
				wind.Publish()
			case lifecycle.Event:
				if e.To == lifecycle.StageDead {
					return
				}
			}
		}
	})
}
<|endoftext|>