<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"path/filepath"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/walker"

	"github.com/minio/sha256-simd"
	"github.com/spf13/cobra"
)

var cmdStats = &cobra.Command{
	Use:   "stats [flags] [snapshot ID] [...]",
	Short: "Scan the repository and show basic statistics",
	Long: `
The "stats" command walks one or multiple snapshots in a repository
and accumulates statistics about the data stored therein. It reports
on the number of unique files and their sizes, according to one of
the counting modes as given by the --mode flag.

It operates on all snapshots matching the selection criteria or all
snapshots if nothing is specified. The special snapshot ID "latest"
is also supported. Some modes make more sense over
just a single snapshot, while others are useful across all snapshots,
depending on what you are trying to calculate.

The modes are:

* restore-size: (default) Counts the size of the restored files.
* files-by-contents: Counts total size of files, where a file is
   considered unique if it has unique contents.
* raw-data: Counts the size of blobs in the repository, regardless of
  how many files reference them.
* blobs-per-file: A combination of files-by-contents and raw-data.

Refer to the online manual for more details about each mode.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runStats(globalOptions, args)
	},
}

// StatsOptions collects all options for the stats command.
type StatsOptions struct {
	// the mode of counting to perform (see consts for available modes)
	countMode string

	// filter snapshots by, if given by user
	Hosts []string
	Tags  restic.TagLists
	Paths []string
}

var statsOptions StatsOptions

func init() {
	cmdRoot.AddCommand(cmdStats)
	f := cmdStats.Flags()
	f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
	f.StringArrayVarP(&statsOptions.Hosts, "host", "H", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
	f.Var(&statsOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
	f.StringArrayVar(&statsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
}

func runStats(gopts GlobalOptions, args []string) error {
	err := verifyStatsInput(gopts, args)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(ctx, repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(ctx); err != nil {
		return err
	}

	if !gopts.JSON {
		Printf("scanning...\n")
	}

	// create a container for the stats (and other needed state)
	stats := &statsContainer{
		uniqueFiles:    make(map[fileID]struct{}),
		fileBlobs:      make(map[string]restic.IDSet),
		blobs:          restic.NewBlobSet(),
		SnapshotsCount: 0,
	}

	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) {
		err = statsWalkSnapshot(ctx, sn, repo, stats)
		if err != nil {
			return fmt.Errorf("error walking snapshot: %v", err)
		}
	}

	if err != nil {
		return err
	}

	if statsOptions.countMode == countModeRawData {
		// the blob handles have been collected, but not yet counted
		for blobHandle := range stats.blobs {
			pbs := repo.Index().Lookup(blobHandle)
			if len(pbs) == 0 {
				return fmt.Errorf("blob %v not found", blobHandle)
			}
			stats.TotalSize += uint64(pbs[0].Length)
			stats.TotalBlobCount++
		}
	}

	if gopts.JSON {
		err = json.NewEncoder(globalOptions.stdout).Encode(stats)
		if err != nil {
			return fmt.Errorf("encoding output: %v", err)
		}
		return nil
	}

	Printf("Stats in %s mode:\n", statsOptions.countMode)
	Printf("Snapshots processed:   %d\n", stats.SnapshotsCount)

	if stats.TotalBlobCount > 0 {
		Printf("   Total Blob Count:   %d\n", stats.TotalBlobCount)
	}
	if stats.TotalFileCount > 0 {
		Printf("   Total File Count:   %d\n", stats.TotalFileCount)
	}
	Printf("         Total Size:   %-5s\n", formatBytes(stats.TotalSize))

	return nil
}

func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, stats *statsContainer) error {
	if snapshot.Tree == nil {
		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
	}

	stats.SnapshotsCount++

	if statsOptions.countMode == countModeRawData {
		// count just the sizes of unique blobs; we don't need to walk the tree
		// ourselves in this case, since a nifty function does it for us
		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
	}

	uniqueInodes := make(map[uint64]struct{})
	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats, uniqueInodes))
	if err != nil {
		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
	}

	return nil
}

func statsWalkTree(repo restic.Repository, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc {
	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
		if nodeErr != nil {
			return true, nodeErr
		}
		if node == nil {
			return true, nil
		}

		if statsOptions.countMode == countModeUniqueFilesByContents || statsOptions.countMode == countModeBlobsPerFile {
			// only count this file if we haven't visited it before
			fid := makeFileIDByContents(node)
			if _, ok := stats.uniqueFiles[fid]; !ok {
				// mark the file as visited
				stats.uniqueFiles[fid] = struct{}{}

				if statsOptions.countMode == countModeUniqueFilesByContents {
					// simply count the size of each unique file (unique by contents only)
					stats.TotalSize += node.Size
					stats.TotalFileCount++
				}
				if statsOptions.countMode == countModeBlobsPerFile {
					// count the size of each unique blob reference, which is
					// by unique file (unique by contents and file path)
					for _, blobID := range node.Content {
						// ensure we have this file (by path) in our map; in this
						// mode, a file is unique by both contents and path
						nodePath := filepath.Join(npath, node.Name)
						if _, ok := stats.fileBlobs[nodePath]; !ok {
							stats.fileBlobs[nodePath] = restic.NewIDSet()
							stats.TotalFileCount++
						}
						if _, ok := stats.fileBlobs[nodePath][blobID]; !ok {
							// is always a data blob since we're accessing it via a file's Content array
							blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob)
							if !found {
								return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID)
							}

							// count the blob's size, then add this blob by this
							// file (path) so we don't double-count it
							stats.TotalSize += uint64(blobSize)
							stats.fileBlobs[nodePath].Insert(blobID)
							// this mode also counts total unique blob _references_ per file
							stats.TotalBlobCount++
						}
					}
				}
			}
		}

		if statsOptions.countMode == countModeRestoreSize {
			// as this is a file in the snapshot, we can simply count its
			// size without worrying about uniqueness, since duplicate files
			// will still be restored
			stats.TotalFileCount++

			// if inodes are present, only count each inode once
			// (hard links do not increase restore size)
			if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
				uniqueInodes[node.Inode] = struct{}{}
				stats.TotalSize += node.Size
			}

			return false, nil
		}

		return true, nil
	}
}

// makeFileIDByContents returns a hash of the blob IDs of the
// node's Content in sequence.
func makeFileIDByContents(node *restic.Node) fileID {
	var bb []byte
	for _, c := range node.Content {
		bb = append(bb, []byte(c[:])...)
	}
	return sha256.Sum256(bb)
}

func verifyStatsInput(gopts GlobalOptions, args []string) error {
	// require a recognized counting mode
	switch statsOptions.countMode {
	case countModeRestoreSize:
	case countModeUniqueFilesByContents:
	case countModeBlobsPerFile:
	case countModeRawData:
	default:
		return fmt.Errorf("unknown counting mode: %s (use the -h flag to get a list of supported modes)", statsOptions.countMode)
	}

	return nil
}

// statsContainer holds information during a walk of a repository
// to collect information about it, as well as state needed
// for a successful and efficient walk.
type statsContainer struct {
	TotalSize      uint64 `json:"total_size"`
	TotalFileCount uint64 `json:"total_file_count"`
	TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
	// holds count of all considered snapshots
	SnapshotsCount int `json:"snapshots_count"`

	// uniqueFiles marks visited files according to their
	// contents (hashed sequence of content blob IDs)
	uniqueFiles map[fileID]struct{}

	// fileBlobs maps a file name (path) to the set of
	// blobs that have been seen as a part of the file
	fileBlobs map[string]restic.IDSet

	// blobs is used to count individual unique blobs,
	// independent of references to files
	blobs restic.BlobSet
}

// fileID is a 256-bit hash that distinguishes unique files.
type fileID [32]byte

const (
	countModeRestoreSize           = "restore-size"
	countModeUniqueFilesByContents = "files-by-contents"
	countModeBlobsPerFile          = "blobs-per-file"
	countModeRawData               = "raw-data"
)
<commit_msg>restic stats: print uncompressed size in mode raw-data<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"path/filepath"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/walker"

	"github.com/minio/sha256-simd"
	"github.com/spf13/cobra"
)

var cmdStats = &cobra.Command{
	Use:   "stats [flags] [snapshot ID] [...]",
	Short: "Scan the repository and show basic statistics",
	Long: `
The "stats" command walks one or multiple snapshots in a repository
and accumulates statistics about the data stored therein. It reports
on the number of unique files and their sizes, according to one of
the counting modes as given by the --mode flag.

It operates on all snapshots matching the selection criteria or all
snapshots if nothing is specified. The special snapshot ID "latest"
is also supported. Some modes make more sense over
just a single snapshot, while others are useful across all snapshots,
depending on what you are trying to calculate.

The modes are:

* restore-size: (default) Counts the size of the restored files.
* files-by-contents: Counts total size of files, where a file is
   considered unique if it has unique contents.
* raw-data: Counts the size of blobs in the repository, regardless of
  how many files reference them.
* blobs-per-file: A combination of files-by-contents and raw-data.

Refer to the online manual for more details about each mode.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runStats(globalOptions, args)
	},
}

// StatsOptions collects all options for the stats command.
type StatsOptions struct {
	// the mode of counting to perform (see consts for available modes)
	countMode string

	// filter snapshots by, if given by user
	Hosts []string
	Tags  restic.TagLists
	Paths []string
}

var statsOptions StatsOptions

func init() {
	cmdRoot.AddCommand(cmdStats)
	f := cmdStats.Flags()
	f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
	f.StringArrayVarP(&statsOptions.Hosts, "host", "H", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
	f.Var(&statsOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
	f.StringArrayVar(&statsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
}

func runStats(gopts GlobalOptions, args []string) error {
	err := verifyStatsInput(gopts, args)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(ctx, repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(ctx); err != nil {
		return err
	}

	if !gopts.JSON {
		Printf("scanning...\n")
	}

	// create a container for the stats (and other needed state)
	stats := &statsContainer{
		uniqueFiles:    make(map[fileID]struct{}),
		fileBlobs:      make(map[string]restic.IDSet),
		blobs:          restic.NewBlobSet(),
		SnapshotsCount: 0,
	}

	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) {
		err = statsWalkSnapshot(ctx, sn, repo, stats)
		if err != nil {
			return fmt.Errorf("error walking snapshot: %v", err)
		}
	}

	if err != nil {
		return err
	}

	if statsOptions.countMode == countModeRawData {
		// the blob handles have been collected, but not yet counted
		for blobHandle := range stats.blobs {
			pbs := repo.Index().Lookup(blobHandle)
			if len(pbs) == 0 {
				return fmt.Errorf("blob %v not found", blobHandle)
			}
			stats.TotalSize += uint64(pbs[0].Length)
			if repo.Config().Version >= 2 {
				if pbs[0].IsCompressed() {
					stats.TotalUncompressedSize += uint64(pbs[0].UncompressedLength)
				} else {
					stats.TotalUncompressedSize += uint64(pbs[0].Length)
				}
			}
			stats.TotalBlobCount++
		}
	}

	if gopts.JSON {
		err = json.NewEncoder(globalOptions.stdout).Encode(stats)
		if err != nil {
			return fmt.Errorf("encoding output: %v", err)
		}
		return nil
	}

	Printf("Stats in %s mode:\n", statsOptions.countMode)
	Printf("    Snapshots processed:  %d\n", stats.SnapshotsCount)

	if stats.TotalBlobCount > 0 {
		Printf("       Total Blob Count:  %d\n", stats.TotalBlobCount)
	}
	if stats.TotalFileCount > 0 {
		Printf("       Total File Count:  %d\n", stats.TotalFileCount)
	}
	if stats.TotalUncompressedSize > 0 {
		Printf("Total Uncompressed Size:  %-5s\n", formatBytes(stats.TotalUncompressedSize))
	}
	Printf("             Total Size:  %-5s\n", formatBytes(stats.TotalSize))

	return nil
}

func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, stats *statsContainer) error {
	if snapshot.Tree == nil {
		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
	}

	stats.SnapshotsCount++

	if statsOptions.countMode == countModeRawData {
		// count just the sizes of unique blobs; we don't need to walk the tree
		// ourselves in this case, since a nifty function does it for us
		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
	}

	uniqueInodes := make(map[uint64]struct{})
	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats, uniqueInodes))
	if err != nil {
		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
	}

	return nil
}

func statsWalkTree(repo restic.Repository, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc {
	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
		if nodeErr != nil {
			return true, nodeErr
		}
		if node == nil {
			return true, nil
		}

		if statsOptions.countMode == countModeUniqueFilesByContents || statsOptions.countMode == countModeBlobsPerFile {
			// only count this file if we haven't visited it before
			fid := makeFileIDByContents(node)
			if _, ok := stats.uniqueFiles[fid]; !ok {
				// mark the file as visited
				stats.uniqueFiles[fid] = struct{}{}

				if statsOptions.countMode == countModeUniqueFilesByContents {
					// simply count the size of each unique file (unique by contents only)
					stats.TotalSize += node.Size
					stats.TotalFileCount++
				}
				if statsOptions.countMode == countModeBlobsPerFile {
					// count the size of each unique blob reference, which is
					// by unique file (unique by contents and file path)
					for _, blobID := range node.Content {
						// ensure we have this file (by path) in our map; in this
						// mode, a file is unique by both contents and path
						nodePath := filepath.Join(npath, node.Name)
						if _, ok := stats.fileBlobs[nodePath]; !ok {
							stats.fileBlobs[nodePath] = restic.NewIDSet()
							stats.TotalFileCount++
						}
						if _, ok := stats.fileBlobs[nodePath][blobID]; !ok {
							// is always a data blob since we're accessing it via a file's Content array
							blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob)
							if !found {
								return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID)
							}

							// count the blob's size, then add this blob by this
							// file (path) so we don't double-count it
							stats.TotalSize += uint64(blobSize)
							stats.fileBlobs[nodePath].Insert(blobID)
							// this mode also counts total unique blob _references_ per file
							stats.TotalBlobCount++
						}
					}
				}
			}
		}

		if statsOptions.countMode == countModeRestoreSize {
			// as this is a file in the snapshot, we can simply count its
			// size without worrying about uniqueness, since duplicate files
			// will still be restored
			stats.TotalFileCount++

			// if inodes are present, only count each inode once
			// (hard links do not increase restore size)
			if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
				uniqueInodes[node.Inode] = struct{}{}
				stats.TotalSize += node.Size
			}

			return false, nil
		}

		return true, nil
	}
}

// makeFileIDByContents returns a hash of the blob IDs of the
// node's Content in sequence.
func makeFileIDByContents(node *restic.Node) fileID {
	var bb []byte
	for _, c := range node.Content {
		bb = append(bb, []byte(c[:])...)
	}
	return sha256.Sum256(bb)
}

func verifyStatsInput(gopts GlobalOptions, args []string) error {
	// require a recognized counting mode
	switch statsOptions.countMode {
	case countModeRestoreSize:
	case countModeUniqueFilesByContents:
	case countModeBlobsPerFile:
	case countModeRawData:
	default:
		return fmt.Errorf("unknown counting mode: %s (use the -h flag to get a list of supported modes)", statsOptions.countMode)
	}

	return nil
}

// statsContainer holds information during a walk of a repository
// to collect information about it, as well as state needed
// for a successful and efficient walk.
type statsContainer struct {
	TotalSize             uint64 `json:"total_size"`
	TotalUncompressedSize uint64 `json:"total_uncompressed_size"`
	TotalFileCount        uint64 `json:"total_file_count"`
	TotalBlobCount        uint64 `json:"total_blob_count,omitempty"`
	// holds count of all considered snapshots
	SnapshotsCount int `json:"snapshots_count"`

	// uniqueFiles marks visited files according to their
	// contents (hashed sequence of content blob IDs)
	uniqueFiles map[fileID]struct{}

	// fileBlobs maps a file name (path) to the set of
	// blobs that have been seen as a part of the file
	fileBlobs map[string]restic.IDSet

	// blobs is used to count individual unique blobs,
	// independent of references to files
	blobs restic.BlobSet
}

// fileID is a 256-bit hash that distinguishes unique files.
type fileID [32]byte

const (
	countModeRestoreSize           = "restore-size"
	countModeUniqueFilesByContents = "files-by-contents"
	countModeBlobsPerFile          = "blobs-per-file"
	countModeRawData               = "raw-data"
)
<|endoftext|>


<commit_before>// Package rebecca is lightweight convenience library for work with database
//
// See github README for instructions: https://github.com/waterlink/rebecca#rebecca
//
// See examples: https://godoc.org/github.com/waterlink/rebecca#pkg-examples
//
// Simple example:
//
//    type Person struct {
//            rebecca.ModelMetadata `tablename:"people"`
//
//            ID   int    `rebecca:"id" rebecca_primary:"true"`
//            Name string `rebecca:"name"`
//            Age  int    `rebecca:"age"`
//    }
//
//    // Create new record
//    p := &Person{Name: "John", Age: 34}
//    if err := rebecca.Save(p); err != nil {
//            // handle error here
//    }
//    fmt.Print(p)
//
//    // Update existing record
//    p.Name = "John Smith"
//    if err := rebecca.Save(p); err != nil {
//            // handle error here
//    }
//    fmt.Print(p)
//
//    // Get record by its primary key
//    p = &Person{}
//    if err := rebecca.Get(25, p); err != nil {
//            // handle error here
//    }
//    fmt.Print(p)
package rebecca

// This file contains thin exported functions only.
//
// For unexported functions see: helpers.go
//
// For Context see: context.go

// Get is for fetching one record
func Get(ID interface{}, record interface{}) error {
	return get(nil, ID, record)
}

// All is for fetching all records
func All(records interface{}) error {
	ctx := &Context{}
	return ctx.All(records)
}

// Where is for fetching specific records
func Where(records interface{}, where string, args ...interface{}) error {
	ctx := &Context{}
	return ctx.Where(records, where, args...)
}

// First is for fetching only one specific record
func First(record interface{}, where string, args ...interface{}) error {
	ctx := &Context{}
	return ctx.First(record, where, args...)
}

// Save is for saving one record (either creating or updating)
func Save(record interface{}) error {
	return save(nil, record)
}

// Remove is for removing the record
func Remove(record interface{}) error {
	return remove(nil, record)
}
<commit_msg>Add convenience rebecca.SetupDriver to not expose details of rebecca/driver package<commit_after>// Package rebecca is lightweight convenience library for work with database
//
// See github README for instructions: https://github.com/waterlink/rebecca#rebecca
//
// See examples: https://godoc.org/github.com/waterlink/rebecca#pkg-examples
//
// Simple example:
//
//    type Person struct {
//            rebecca.ModelMetadata `tablename:"people"`
//
//            ID   int    `rebecca:"id" rebecca_primary:"true"`
//            Name string `rebecca:"name"`
//            Age  int    `rebecca:"age"`
//    }
//
//    // Create new record
//    p := &Person{Name: "John", Age: 34}
//    if err := rebecca.Save(p); err != nil {
//            // handle error here
//    }
//    fmt.Print(p)
//
//    // Update existing record
//    p.Name = "John Smith"
//    if err := rebecca.Save(p); err != nil {
//            // handle error here
//    }
//    fmt.Print(p)
//
//    // Get record by its primary key
//    p = &Person{}
//    if err := rebecca.Get(25, p); err != nil {
//            // handle error here
//    }
//    fmt.Print(p)
package rebecca

import "github.com/waterlink/rebecca/driver"

// This file contains thin exported functions only.
//
// For unexported functions see: helpers.go
//
// For Context see: context.go

// SetupDriver is for configuring database driver
func SetupDriver(d driver.Driver) {
	driver.SetupDriver(d)
}

// Get is for fetching one record
func Get(ID interface{}, record interface{}) error {
	return get(nil, ID, record)
}

// All is for fetching all records
func All(records interface{}) error {
	ctx := &Context{}
	return ctx.All(records)
}

// Where is for fetching specific records
func Where(records interface{}, where string, args ...interface{}) error {
	ctx := &Context{}
	return ctx.Where(records, where, args...)
}

// First is for fetching only one specific record
func First(record interface{}, where string, args ...interface{}) error {
	ctx := &Context{}
	return ctx.First(record, where, args...)
}

// Save is for saving one record (either creating or updating)
func Save(record interface{}) error {
	return save(nil, record)
}

// Remove is for removing the record
func Remove(record interface{}) error {
	return remove(nil, record)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"net/http"
	"os"
	"strconv"

	"golang.org/x/net/http2"
	"golang.org/x/net/http2/hpack"

	"github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/protocol"
)

var supportedVersions = map[protocol.VersionNumber]bool{
	30: true,
	32: true,
}

func main() {
	path := os.Getenv("GOPATH") + "/src/github.com/lucas-clemente/quic-go/example/"

	server, err := quic.NewServer(path+"cert.der", path+"key.der", handleStream)
	if err != nil {
		panic(err)
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello world!"))
	})

	err = server.ListenAndServe("localhost:6121")
	if err != nil {
		panic(err)
	}
}

type responseWriter struct {
	session      *quic.Session
	dataStreamID protocol.StreamID
	headerStream *quic.Stream
	dataStream   *quic.Stream

	header        http.Header
	headerWritten bool
}

func (w *responseWriter) Header() http.Header {
	return w.header
}

func (w *responseWriter) WriteHeader(status int) {
	w.headerWritten = true

	var headers bytes.Buffer
	enc := hpack.NewEncoder(&headers)
	enc.WriteField(hpack.HeaderField{Name: ":status", Value: strconv.Itoa(status)})
	// enc.WriteField(hpack.HeaderField{Name: "content-length", Value: strconv.Itoa(len(p))})
	// enc.WriteField(hpack.HeaderField{Name: "content-type", Value: http.DetectContentType(p)})

	for k, v := range w.header {
		enc.WriteField(hpack.HeaderField{Name: k, Value: v[0]})
	}

	fmt.Printf("responding with %d %#v\n", status, w.header)
	h2framer := http2.NewFramer(w.headerStream, nil)
	h2framer.WriteHeaders(http2.HeadersFrameParam{
		StreamID:      uint32(w.dataStreamID),
		EndHeaders:    true,
		BlockFragment: headers.Bytes(),
	})
}

func (w *responseWriter) Write(p []byte) (int, error) {
	if !w.headerWritten {
		w.WriteHeader(200)
	}

	if len(p) != 0 {
		dataStream, err := w.session.NewStream(w.dataStreamID)
		if err != nil {
			return 0, fmt.Errorf("error creating data stream: %s\n", err.Error())
		}
		defer dataStream.Close()
		return dataStream.Write(p)
	}

	return 0, nil
}

func handleStream(session *quic.Session, headerStream *quic.Stream) {
	hpackDecoder := hpack.NewDecoder(4096, nil)
	h2framer := http2.NewFramer(nil, headerStream)

	go func() {
		for {
			h2frame, err := h2framer.ReadFrame()
			if err != nil {
				fmt.Printf("invalid http2 frame: %s\n", err.Error())
				continue
			}
			h2headersFrame := h2frame.(*http2.HeadersFrame)
			if !h2headersFrame.HeadersEnded() {
				fmt.Printf("http2 header continuation not implemented")
				continue
			}
			headers, err := hpackDecoder.DecodeFull(h2headersFrame.HeaderBlockFragment())
			if err != nil {
				fmt.Printf("invalid http2 headers encoding: %s\n", err.Error())
				continue
			}

			headersMap := map[string]string{}
			for _, h := range headers {
				headersMap[h.Name] = h.Value
			}

			fmt.Printf("Request: %s %s://%s%s on stream %d\n", headersMap[":method"], headersMap[":scheme"], headersMap[":authority"], headersMap[":path"], h2headersFrame.StreamID)

			req, err := http.NewRequest(headersMap[":method"], headersMap[":path"], nil)
			if err != nil {
				fmt.Printf("invalid http2 frame: %s\n", err.Error())
				continue
			}

			responseWriter := &responseWriter{
				header:       http.Header{},
				headerStream: headerStream,
				dataStreamID: protocol.StreamID(h2headersFrame.StreamID),
				session:      session,
			}

			go http.DefaultServeMux.ServeHTTP(responseWriter, req)
		}
	}()
}
<commit_msg>fix content-length handling and multiple writes in http2 responsewriter<commit_after>package main

import (
	"bytes"
	"fmt"
	"net/http"
	"os"
	"strconv"

	"golang.org/x/net/http2"
	"golang.org/x/net/http2/hpack"

	"github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/protocol"
)

var supportedVersions = map[protocol.VersionNumber]bool{
	30: true,
	32: true,
}

func main() {
	path := os.Getenv("GOPATH") + "/src/github.com/lucas-clemente/quic-go/example/"

	server, err := quic.NewServer(path+"cert.der", path+"key.der", handleStream)
	if err != nil {
		panic(err)
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello world!"))
	})

	err = server.ListenAndServe("localhost:6121")
	if err != nil {
		panic(err)
	}
}

type responseWriter struct {
	session      *quic.Session
	dataStreamID protocol.StreamID
	headerStream *quic.Stream
	dataStream   *quic.Stream

	header        http.Header
	headerWritten bool

	bytesWritten  int
	contentLength int
}

func (w *responseWriter) Header() http.Header {
	return w.header
}

func (w *responseWriter) WriteHeader(status int) {
	w.headerWritten = true

	var headers bytes.Buffer
	enc := hpack.NewEncoder(&headers)
	enc.WriteField(hpack.HeaderField{Name: ":status", Value: strconv.Itoa(status)})
	// enc.WriteField(hpack.HeaderField{Name: "content-length", Value: strconv.Itoa(len(p))})
	// enc.WriteField(hpack.HeaderField{Name: "content-type", Value: http.DetectContentType(p)})

	for k, v := range w.header {
		enc.WriteField(hpack.HeaderField{Name: k, Value: v[0]})
	}

	fmt.Printf("Responding with %d %#v\n", status, w.header)
	h2framer := http2.NewFramer(w.headerStream, nil)
	h2framer.WriteHeaders(http2.HeadersFrameParam{
		StreamID:      uint32(w.dataStreamID),
		EndHeaders:    true,
		BlockFragment: headers.Bytes(),
	})

	w.contentLength, _ = strconv.Atoi(w.header.Get("content-length"))
}

func (w *responseWriter) Write(p []byte) (int, error) {
	if !w.headerWritten {
		w.WriteHeader(200)
	}

	if len(p) != 0 {
		if w.dataStream == nil {
			var err error
			w.dataStream, err = w.session.NewStream(w.dataStreamID)
			if err != nil {
				return 0, fmt.Errorf("error creating data stream: %s\n", err.Error())
			}
		}

		n, err := w.dataStream.Write(p)
		w.bytesWritten += n

		if w.bytesWritten >= w.contentLength {
			defer w.dataStream.Close()
		}

		return n, err
	}

	return 0, nil
}

func handleStream(session *quic.Session, headerStream *quic.Stream) {
	hpackDecoder := hpack.NewDecoder(4096, nil)
	h2framer := http2.NewFramer(nil, headerStream)

	go func() {
		for {
			h2frame, err := h2framer.ReadFrame()
			if err != nil {
				fmt.Printf("invalid http2 frame: %s\n", err.Error())
				continue
			}
			h2headersFrame := h2frame.(*http2.HeadersFrame)
			if !h2headersFrame.HeadersEnded() {
				fmt.Printf("http2 header continuation not implemented")
				continue
			}
			headers, err := hpackDecoder.DecodeFull(h2headersFrame.HeaderBlockFragment())
			if err != nil {
				fmt.Printf("invalid http2 headers encoding: %s\n", err.Error())
				continue
			}

			headersMap := map[string]string{}
			for _, h := range headers {
				headersMap[h.Name] = h.Value
			}

			fmt.Printf("Request: %s %s://%s%s on stream %d\n", headersMap[":method"], headersMap[":scheme"], headersMap[":authority"], headersMap[":path"], h2headersFrame.StreamID)

			req, err := http.NewRequest(headersMap[":method"], headersMap[":path"], nil)
			if err != nil {
				fmt.Printf("invalid http2 frame: %s\n", err.Error())
				continue
			}

			responseWriter := &responseWriter{
				header:       http.Header{},
				headerStream: headerStream,
				dataStreamID: protocol.StreamID(h2headersFrame.StreamID),
				session:      session,
			}

			go http.DefaultServeMux.ServeHTTP(responseWriter, req)
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bytes"
	"context"
	"crypto/sha1"
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

var NetlifyHookURL = os.Getenv("NETLIFY_DEPLOY_HOOK")

func create_comparison(r *http.Request) (interface{}, error) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("Could not read request body")
	}

	params := &struct {
		Title  string
		Images [][]*image
	}{}
	err = json.Unmarshal(body, &params)
	if err != nil {
		return nil, fmt.Errorf("Could not decode request body")
	}

	// Step 1: Decode & sanitize the request
	images := map[string][]*image{}
	paths := []interface{}{}
	for _, row := range params.Images {
		names := map[string]struct{}{}
		for _, img := range row {
			if _, ok := names[img.Name]; ok {
				return nil, fmt.Errorf("Duplicate Name")
			}
			if !hashRe.MatchString(img.Sha) {
				return nil, fmt.Errorf("Invalid hash: %v", img.Sha)
			}

			img.Path = fmt.Sprintf("%s/%s/%s", img.Sha[0:2], img.Sha[2:4], img.Sha[4:])
			names[img.Name] = struct{}{}
			images[img.Path] = append(images[img.Path], img)
			paths = append(paths, img.Path)
		}
	}
	if len(paths) == 0 {
		return nil, fmt.Errorf("No images to compare")
	}

	// Step 2: Look up all the images we're comparing
	rows, err := db.Query(fmt.Sprintf("SELECT `id`, `path`, `thumb`, `format` FROM `images` WHERE `path` IN (?%s)", strings.Repeat(",?", len(paths)-1)), paths...)
	if err != nil {
		return nil, fmt.Errorf("Invalid SQL: %v", err)
	}
	defer rows.Close()
	for rows.Next() {
		var (
			id     string
			path   string
			thumb  string
			format string
		)
		err = rows.Scan(&id, &path, &thumb, &format)
		if err != nil {
			return nil, fmt.Errorf("Invalid SQL: %v", err)
		}
		for _, img := range images[path] {
			img.ID = id
			img.Thumb = thumb
			img.Format = format
		}
	}
	err = rows.Err()
	if err != nil {
		return nil, fmt.Errorf("Invalid SQL: %v", err)
	}

	// Step 3: Wait for images to finish processing if they're not done yet (they should be)
	time.Sleep(10 * time.Second) // TODO: use webhooks for this detection, should speed things up by 10 seconds and make it more reliable

	// Step 4: Download all the images (to make the ZIP file)
	eg, ctx := errgroup.WithContext(context.Background())
	for _, row := range params.Images {
		for _, img := range row {
			img := img
			eg.Go(func() error {
				buf := aws.NewWriteAtBuffer([]byte{})
				_, err := downloader.DownloadWithContext(ctx, buf, &s3.GetObjectInput{
					Bucket: aws.String("diff.pics"),
					Key:    aws.String("images/" + img.Path),
				})
				img.Data = buf.Bytes()
				return errors.Wrap(err, img.Path)
			})
		}
	}
	if err := eg.Wait(); err != nil {
		return nil, fmt.Errorf("Failed to download images: %v", err)
	}

	// Step 5: Make the zip file
	buf := &bytes.Buffer{}
	zw := zip.NewWriter(buf)
	for i, row := range params.Images {
		for _, img := range row {
			f, err := zw.Create(fmt.Sprintf("%d/%s.%s", i, img.Name, img.Format))
			if err != nil {
				return nil, fmt.Errorf("Failed to create zip file: %v", err)
			}
			_, err = f.Write(img.Data)
			if err != nil {
				return nil, fmt.Errorf("Failed to create zip file: %v", err)
			}
		}
	}
	err = zw.Close()
	if err != nil {
		return nil, fmt.Errorf("Failed to create zip file: %v", err)
	}

	// Step 6: Upload the zip file
	zipName := fmt.Sprintf("%X", sha1.Sum(buf.Bytes()))
	zipName = fmt.Sprintf("%s/%s/%s", zipName[0:2], zipName[2:4], zipName[4:])
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String("diff.pics"),
		Key:    aws.String("zips/" + zipName),
		Body:   buf,
	})
	if err != nil {
		return nil, fmt.Errorf("Failed to upload zip file: %v", err)
	}

	// Step 7: Find a key for the comparison
	var key string
	for keyLen := 12; keyLen <= 51; keyLen++ {
		key = randString(keyLen)

		var _ignore string
		err = db.QueryRow("SELECT `id` FROM `comparisons` WHERE `key` = ?", key).Scan(&_ignore)
		if err == sql.ErrNoRows {
			break
		}
	}
	if len(key) > 50 {
		return nil, fmt.Errorf("Couldn't generate key")
	}

	// Step 8: Add comparison to the database
	result, err := db.Exec("INSERT INTO `comparisons`(`key`, `title`, `zip`) VALUES(?,?,?)", key, params.Title, zipName)
	if err != nil {
		return nil, fmt.Errorf("Failed to insert comparison: %v", err)
	}
	comparisonID, err := result.LastInsertId()
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch comparison id: %v", err)
	}

	// Step 9: Add images to the database
	numImages := 0
	args := []interface{}{}
	for r, row := range params.Images {
		for c, img := range row {
			numImages++
			args = append(args, comparisonID, r, c, img.ID, img.Name)
		}
	}
	rowTemplate := "(?,?,?,?,?)"
	_, err = db.Exec(fmt.Sprintf("INSERT INTO `comparison_images`(`comparison_id`, `row`, `column`, `image_id`, `name`) VALUES %s%s", rowTemplate, strings.Repeat(","+rowTemplate, numImages-1)), args...)
	if err != nil {
		return nil, fmt.Errorf("Failed to insert comparison images: %v", err)
	}

	// Step 10: Run a full build
	for {
		if _, err := http.Post(NetlifyHookURL, "", nil); err == nil {
			break
		}
	}
	var deployID string
	for {
		time.Sleep(2 * time.Second)
		resp, err := http.Get("https://api.netlify.com/api/v1/sites/diff.pics/deploys?access_token=" + AccessToken)
		if err != nil {
			continue
		}
		defer resp.Body.Close()
		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		deployResp := []struct {
			DeployID string `json:"id"`
		}{}
		err = json.Unmarshal(body, &deployResp)
		if err != nil {
			continue
		}

		deployID = deployResp[0].DeployID
		break
	}
	for {
		if err = fullBuild(); err == nil {
			break
		}
	}

	digest := &struct {
		State        string `json:"state"`
		ErrorMessage string `json:"error_message"`
	}{}
	for digest.State != "prepared" && digest.State != "ready" {
		// Check for errors in preproccessing
		if digest.State == "error" {
			return nil, fmt.Errorf("Deploy failed: %s -- Your comparison will eventually be at https://diff.pics/%s/1", digest.ErrorMessage, key)
		}
		time.Sleep(2 * time.Second)
		resp, err := http.Get("https://api.netlify.com/api/v1/deploys/" + deployID + "?access_token=" + AccessToken)
		if err != nil {
			continue
		}
		defer resp.Body.Close()
		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		err = json.Unmarshal(body, &digest)
		if err != nil {
			continue
		}
	}

	// Step 11: return the url to go to
	return map[string]string{"URL": fmt.Sprintf("/%s/1", key)}, nil
}

type image struct {
	Name string
	Sha  string

	ID     string
	Path   string
	Thumb  string
	Format string

	Data []byte
}

const randStringChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"

func init() {
	rand.Seed(time.Now().UnixNano())
}
func randString(length int) string {
	b := make([]byte, length)

	numBytes := int(1 + math.Log2(float64(len(randStringChars)-1)))
	mask := int64(1<<uint(numBytes) - 1)
	charsPerRand := 63 / numBytes

	for i := 0; i < length; i += charsPerRand {
		for j, r := 0, rand.Int63(); j < charsPerRand && i+j < length; j++ {
			idx := int((r >> uint(j*numBytes)) & mask)
			b[i+j] = randStringChars[idx]
		}
	}

	return string(b)
}
<commit_msg>Fixes<commit_after>package main

import (
	"archive/zip"
	"bytes"
	"context"
	"crypto/sha1"
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

var NetlifyHookURL = os.Getenv("NETLIFY_DEPLOY_HOOK")

func create_comparison(r *http.Request) (interface{}, error) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("Could not read request body")
	}

	params := &struct {
		Title  string
		Images [][]*image
	}{}
	err = json.Unmarshal(body, &params)
	if err != nil {
		return nil, fmt.Errorf("Could not decode request body")
	}

	// Step 1: Decode & sanitize the request
	images := map[string][]*image{}
	paths := []interface{}{}
	for _, row := range params.Images {
		names := map[string]struct{}{}
		for _, img := range row {
			if _, ok := names[img.Name]; ok {
				return nil, fmt.Errorf("Duplicate Name")
			}
			if !hashRe.MatchString(img.Sha) {
				return nil, fmt.Errorf("Invalid hash: %v", img.Sha)
			}

			img.Path = fmt.Sprintf("%s/%s/%s", img.Sha[0:2], img.Sha[2:4], img.Sha[4:])
			names[img.Name] = struct{}{}
			images[img.Path] = append(images[img.Path], img)
			paths = append(paths, img.Path)
		}
	}
	if len(paths) == 0 {
		return nil, fmt.Errorf("No images to compare")
	}

	// Step 3: Wait for images to finish processing if they're not done yet (they should be)
	time.Sleep(10 * time.Second) // TODO: use webhooks for this detection, should speed things up by 10 seconds and make it more reliable

	// Step 2: Look up all the images we're comparing
	rows, err := db.Query(fmt.Sprintf("SELECT `id`, `path`, `thumb`, `format` FROM `images` WHERE `path` IN (?%s)", strings.Repeat(",?", len(paths)-1)), paths...)
	if err != nil {
		return nil, fmt.Errorf("Invalid SQL: %v", err)
	}
	defer rows.Close()
	for rows.Next() {
		var (
			id     string
			path   string
			thumb  string
			format string
		)
		err = rows.Scan(&id, &path, &thumb, &format)
		if err != nil {
			return nil, fmt.Errorf("Invalid SQL: %v", err)
		}
		for _, img := range images[path] {
			img.ID = id
			img.Thumb = thumb
			img.Format = format
		}
	}
	err = rows.Err()
	if err != nil {
		return nil, fmt.Errorf("Invalid SQL: %v", err)
	}

	// Step 4: Download all the images (to make the ZIP file)
	eg, ctx := errgroup.WithContext(context.Background())
	for _, row := range params.Images {
		for _, img := range row {
			img := img
			if img.ID == "" {
				return nil, fmt.Errorf("Couldn't lookup image %q", img.Sha)
			}
			eg.Go(func() error {
				buf := aws.NewWriteAtBuffer([]byte{})
				_, err := downloader.DownloadWithContext(ctx, buf, &s3.GetObjectInput{
					Bucket: aws.String("diff.pics"),
					Key:    aws.String("images/" + img.Path),
				})
				img.Data = buf.Bytes()
				return errors.Wrap(err, img.Path)
			})
		}
	}
	if err := eg.Wait(); err != nil {
		return nil, fmt.Errorf("Failed to download images: %v", err)
	}

	// Step 5: Make the zip file
	buf := &bytes.Buffer{}
	zw := zip.NewWriter(buf)
	for i, row := range params.Images {
		for _, img := range row {
			f, err := zw.Create(fmt.Sprintf("%d/%s.%s", i, img.Name, img.Format))
			if err != nil {
				return nil, fmt.Errorf("Failed to create zip file: %v", err)
			}
			_, err = f.Write(img.Data)
			if err != nil {
				return nil, fmt.Errorf("Failed to create zip file: %v", err)
			}
		}
	}
	err = zw.Close()
	if err != nil {
		return nil, fmt.Errorf("Failed to create zip file: %v", err)
	}

	// Step 6: Upload the zip file
	zipName := fmt.Sprintf("%X", sha1.Sum(buf.Bytes()))
	zipName = fmt.Sprintf("%s/%s/%s", zipName[0:2], zipName[2:4], zipName[4:])
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String("diff.pics"),
		Key:    aws.String("zips/" + zipName),
		Body:   buf,
	})
	if err != nil {
		return nil, fmt.Errorf("Failed to upload zip file: %v", err)
	}

	// Step 7: Find a key for the comparison
	var key string
	for keyLen := 12; keyLen <= 51; keyLen++ {
		key = randString(keyLen)

		var _ignore string
		err = db.QueryRow("SELECT `id` FROM `comparisons` WHERE `key` = ?", key).Scan(&_ignore)
		if err == sql.ErrNoRows {
			break
		}
	}
	if len(key) > 50 {
		return nil, fmt.Errorf("Couldn't generate key")
	}

	// Step 8: Add comparison to the database
	result, err := db.Exec("INSERT INTO `comparisons`(`key`, `title`, `zip`) VALUES(?,?,?)", key, params.Title, zipName)
	if err != nil {
		return nil, fmt.Errorf("Failed to insert comparison: %v", err)
	}
	comparisonID, err := result.LastInsertId()
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch comparison id: %v", err)
	}

	// Step 9: Add images to the database
	numImages := 0
	args := []interface{}{}
	for r, row := range params.Images {
		for c, img := range row {
			numImages++
			args = append(args, comparisonID, r, c, img.ID, img.Name)
		}
	}
	rowTemplate := "(?,?,?,?,?)"
	_, err = db.Exec(fmt.Sprintf("INSERT INTO `comparison_images`(`comparison_id`, `row`, `column`, `image_id`, `name`) VALUES %s%s", rowTemplate, strings.Repeat(","+rowTemplate, numImages-1)), args...)
	if err != nil {
		return nil, fmt.Errorf("Failed to insert comparison images: %v", err)
	}

	// Step 10: Run a full build
	for {
		if _, err := http.Post(NetlifyHookURL, "", nil); err == nil {
			break
		}
	}
	var deployID string
	for {
		time.Sleep(2 * time.Second)
		resp, err := http.Get("https://api.netlify.com/api/v1/sites/diff.pics/deploys?access_token=" + AccessToken)
		if err != nil {
			continue
		}
		defer resp.Body.Close()
		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		deployResp := []struct {
			DeployID string `json:"id"`
		}{}
		err = json.Unmarshal(body, &deployResp)
		if err != nil {
			continue
		}

		deployID = deployResp[0].DeployID
		break
	}
	for {
		if err = fullBuild(); err == nil {
			break
		}
	}

	digest := &struct {
		State        string `json:"state"`
		ErrorMessage string `json:"error_message"`
	}{}
	for digest.State != "prepared" && digest.State != "ready" {
		// Check for errors in preproccessing
		if digest.State == "error" {
			return nil, fmt.Errorf("Deploy failed: %s -- Your comparison will eventually be at https://diff.pics/%s/1", digest.ErrorMessage, key)
		}
		time.Sleep(2 * time.Second)
		resp, err := http.Get("https://api.netlify.com/api/v1/deploys/" + deployID + "?access_token=" + AccessToken)
		if err != nil {
			continue
		}
		defer resp.Body.Close()
		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		err = json.Unmarshal(body, &digest)
		if err != nil {
			continue
		}
	}

	// Step 11: return the url to go to
	return map[string]string{"URL": fmt.Sprintf("/%s/1", key)}, nil
}

type image struct {
	Name string
	Sha  string

	ID     string
	Path   string
	Thumb  string
	Format string

	Data []byte
}

const randStringChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"

func init() {
	rand.Seed(time.Now().UnixNano())
}
func randString(length int) string {
	b := make([]byte, length)

	numBytes := int(1 + math.Log2(float64(len(randStringChars)-1)))
	mask := int64(1<<uint(numBytes) - 1)
	charsPerRand := 63 / numBytes

	for i := 0; i < length; i += charsPerRand {
		for j, r := 0, rand.Int63(); j < charsPerRand && i+j < length; j++ {
			idx := int((r >> uint(j*numBytes)) & mask)
			b[i+j] = randStringChars[idx]
		}
	}

	return string(b)
}
<|endoftext|>


<commit_before>package main

import (
    "../../cmd"

    "fmt"
    )

func main() {
    commander := &cmd.Cmd{}
    commander.Init()

    commander.Commands["ls"] = func(line string) (stop bool) {
        fmt.Println("listing stuff")
        return
    }

    commander.Commands["exit"] = func(line string) (stop bool) {
        fmt.Println("goodbye!")
        return true
    }

    commander.CmdLoop()
}
<commit_msg>Use the correct import path<commit_after>package main

import (
    "github.com/gobs/cmd"

    "fmt"
    )

func main() {
    commander := &cmd.Cmd{}
    commander.Init()

    commander.Commands["ls"] = func(line string) (stop bool) {
        fmt.Println("listing stuff")
        return
    }

    commander.Commands["exit"] = func(line string) (stop bool) {
        fmt.Println("goodbye!")
        return true
    }

    commander.CmdLoop()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/RobotsAndPencils/buford"
)

func main() {
	var deviceToken, filename, password string

	flag.StringVar(&deviceToken, "d", "", "Device token")
	flag.StringVar(&filename, "c", "", "Path to p12 certificate file")
	flag.StringVar(&password, "p", "", "Password for p12 file.")
	flag.Parse()

	cert, err := buford.LoadCert(filename, password)
	if err != nil {
		log.Fatal(err)
	}

	service := buford.Service{
		Client: buford.NewClient(cert),
		Host:   buford.Sandbox,
	}

	err = service.Push(deviceToken, buford.Headers{}, []byte(`{ "aps" : { "alert" : "Hello HTTP/2" } }`))
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>example: flag to use live environment<commit_after>package main

import (
	"flag"
	"log"

	"github.com/RobotsAndPencils/buford"
)

func main() {
	var deviceToken, filename, password, environment string

	flag.StringVar(&deviceToken, "d", "", "Device token")
	flag.StringVar(&filename, "c", "", "Path to p12 certificate file")
	flag.StringVar(&password, "p", "", "Password for p12 file.")
	flag.StringVar(&environment, "e", "development", "Environment")
	flag.Parse()

	cert, err := buford.LoadCert(filename, password)
	if err != nil {
		log.Fatal(err)
	}

	service := buford.Service{
		Client: buford.NewClient(cert),
		Host:   buford.Sandbox,
	}
	if environment == "production" {
		service.Host = buford.Live
	}

	err = service.Push(deviceToken, buford.Headers{}, []byte(`{ "aps" : { "alert" : "Hello HTTP/2" } }`))
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package notifier

import (
	"bytes"
	"fmt"
	"os"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/fetcher"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/sendgrid/sendgrid-go"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
	"github.com/uber-go/zap"
)

var lessonFetcher *fetcher.TeacherLessonFetcher

func init() {
	lessonFetcher = fetcher.NewTeacherLessonFetcher(nil, logger.AppLogger)
}

type Notifier struct {
	db            *gorm.DB
	dryRun        bool
	lessonService *model.LessonService
	teachers      map[uint32]*model.Teacher
}

func NewNotifier(db *gorm.DB, dryRun bool) *Notifier {
	return &Notifier{
		db:       db,
		dryRun:   dryRun,
		teachers: make(map[uint32]*model.Teacher, 1000),
	}
}

func (n *Notifier) SendNotification(user *model.User) error {
	followingTeacherService := model.NewFollowingTeacherService(n.db)
	n.lessonService = model.NewLessonService(n.db)

	teacherIDs, err := followingTeacherService.FindTeacherIDsByUserID(user.ID)
	if err != nil {
		return errors.Wrapperf(err, "Failed to FindTeacherIDsByUserID(): userID=%v", user.ID)
	}

	availableLessonsPerTeacher := make(map[uint32][]*model.Lesson, 1000)
	allFetchedLessons := make([]*model.Lesson, 0, 5000)
	for _, teacherID := range teacherIDs {
		teacher, fetchedLessons, newAvailableLessons, err := n.fetchAndExtractNewAvailableLessons(teacherID)
		if err != nil {
			switch err.(type) {
			case *errors.NotFound:
				// TODO: update teacher table flag
				// TODO: Not need to log
				logger.AppLogger.Warn("Cannot fetch teacher", zap.Uint("teacherID", uint(teacherID)))
				continue
			default:
				return err
			}
		}

		allFetchedLessons = append(allFetchedLessons, fetchedLessons...)
		n.teachers[teacherID] = teacher
		if len(newAvailableLessons) > 0 {
			availableLessonsPerTeacher[teacherID] = newAvailableLessons
		}
	}

	if err := n.sendNotificationToUser(user, availableLessonsPerTeacher); err != nil {
		return err
	}

	if !n.dryRun {
		n.lessonService.UpdateLessons(allFetchedLessons)
	}

	time.Sleep(500 * time.Millisecond)

	return nil
}

// Returns teacher, fetchedLessons, newAvailableLessons, error
func (n *Notifier) fetchAndExtractNewAvailableLessons(teacherID uint32) (
	*model.Teacher, []*model.Lesson, []*model.Lesson, error,
) {
	teacher, fetchedLessons, err := lessonFetcher.Fetch(teacherID)
	if err != nil {
		logger.AppLogger.Error(
			"TeacherLessonFetcher.Fetch",
			zap.Uint("teacherID", uint(teacherID)), zap.Error(err),
		)
		return nil, nil, nil, err
	}
	logger.AppLogger.Info(
		"TeacherLessonFetcher.Fetch",
		zap.Uint("teacherID", uint(teacher.ID)),
		zap.String("teacherName", teacher.Name),
		zap.Int("fetchedLessons", len(fetchedLessons)),
	)

	//fmt.Printf("fetchedLessons ---\n")
	//for _, l := range fetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	now := time.Now()
	fromDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, config.LocalTimezone())
	toDate := fromDate.Add(24 * 6 * time.Hour)
	lastFetchedLessons, err := n.lessonService.FindLessons(teacher.ID, fromDate, toDate)
	if err != nil {
		return nil, nil, nil, err
	}
	//fmt.Printf("lastFetchedLessons ---\n")
	//for _, l := range lastFetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	newAvailableLessons := n.lessonService.GetNewAvailableLessons(lastFetchedLessons, fetchedLessons)
	//fmt.Printf("newAvailableLessons ---\n")
	//for _, l := range newAvailableLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}
	return teacher, fetchedLessons, newAvailableLessons, nil
}

func (n *Notifier) sendNotificationToUser(
	user *model.User,
	lessonsPerTeacher map[uint32][]*model.Lesson,
) error {
	lessonsCount := 0
	var teacherIDs []int
	for teacherID, lessons := range lessonsPerTeacher {
		teacherIDs = append(teacherIDs, int(teacherID))
		lessonsCount += len(lessons)
	}
	if lessonsCount == 0 {
		// Don't send notification
		return nil
	}

	sort.Ints(teacherIDs)
	var teacherIDs2 []uint32
	var teacherNames []string
	for _, id := range teacherIDs {
		teacherIDs2 = append(teacherIDs2, uint32(id))
		teacherNames = append(teacherNames, n.teachers[uint32(id)].Name)
	}

	t := template.New("email")
	t = template.Must(t.Parse(getEmailTemplateJP()))
	type TemplateData struct {
		TeacherIDs        []uint32
		Teachers          map[uint32]*model.Teacher
		LessonsPerTeacher map[uint32][]*model.Lesson
		WebURL            string
	}
	data := &TemplateData{
		TeacherIDs:        teacherIDs2,
		Teachers:          n.teachers,
		LessonsPerTeacher: lessonsPerTeacher,
		WebURL:            config.WebURL(),
	}

	var body bytes.Buffer
	if err := t.Execute(&body, data); err != nil {
		return errors.InternalWrapf(err, "Failed to execute template.")
	}
	//fmt.Printf("--- mail ---\n%s", body.String())

	//subject := "Schedule of teacher " + strings.Join(teacherNames, ", ")
	subject := strings.Join(teacherNames, ", ") + "の空きレッスンがあります"
	sender := &EmailNotificationSender{}
	return sender.Send(user, subject, body.String())
}

func getEmailTemplateJP() string {
	return strings.TrimSpace(`
{{- range $teacherID := .TeacherIDs }}
{{- $teacher := index $.Teachers $teacherID -}}
--- {{ $teacher.Name }} ---
  {{- $lessons := index $.LessonsPerTeacher $teacherID }}
  {{- range $lesson := $lessons }}
{{ $lesson.Datetime.Format "2006-01-02 15:04" }}
  {{- end }}

レッスンの予約はこちらから:
PC
Mobile

{{ end }}
空きレッスンの通知の解除はこちら
	`)
}

func getEmailTemplateEN() string {
	return strings.TrimSpace(`
{{- range $teacherID := .TeacherIDs }}
{{- $teacher := index $.Teachers $teacherID -}}
--- {{ $teacher.Name }} ---
  {{- $lessons := index $.LessonsPerTeacher $teacherID }}
  {{- range $lesson := $lessons }}
{{ $lesson.Datetime.Format "2006-01-02 15:04" }}
  {{- end }}

Reserve here:
PC
Mobile
{{ end }}
Click here if you want to stop notification of the teacher.
	`)
}

type NotificationSender interface {
	Send(user *model.User, subject, body string) error
}

type EmailNotificationSender struct{}

func (s *EmailNotificationSender) Send(user *model.User, subject, body string) error {
	from := mail.NewEmail("lekcije", "lekcije@lekcije.com")
	to := mail.NewEmail(user.Name, user.Email.Raw())
	content := mail.NewContent("text/html", strings.Replace(body, "\n", "
", -1))
	m := mail.NewV3MailInit(from, subject, to, content)

	req := sendgrid.GetRequest(
		os.Getenv("SENDGRID_API_KEY"),
		"/v3/mail/send",
		"https://api.sendgrid.com",
	)
	req.Method = "POST"
	req.Body = mail.GetRequestBody(m)
	resp, err := sendgrid.API(req)
	if err != nil {
		return errors.InternalWrapf(err, "Failed to send email by sendgrid")
	}
	if resp.StatusCode >= 300 {
		message := fmt.Sprintf(
			"Failed to send email by sendgrid: statusCode=%v, body=%v",
			resp.StatusCode, strings.Replace(resp.Body, "\n", "\\n", -1),
		)
		logger.AppLogger.Error(message)
		return errors.InternalWrapf(
			err,
			"Failed to send email by sendgrid: statusCode=%v",
			resp.StatusCode,
		)
	}

	return nil
}
<commit_msg>Change URL: / -> /me<commit_after>package notifier

import (
	"bytes"
	"fmt"
	"os"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/fetcher"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/sendgrid/sendgrid-go"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
	"github.com/uber-go/zap"
)

var lessonFetcher *fetcher.TeacherLessonFetcher

func init() {
	lessonFetcher = fetcher.NewTeacherLessonFetcher(nil, logger.AppLogger)
}

type Notifier struct {
	db            *gorm.DB
	dryRun        bool
	lessonService *model.LessonService
	teachers      map[uint32]*model.Teacher
}

func NewNotifier(db *gorm.DB, dryRun bool) *Notifier {
	return &Notifier{
		db:       db,
		dryRun:   dryRun,
		teachers: make(map[uint32]*model.Teacher, 1000),
	}
}

func (n *Notifier) SendNotification(user *model.User) error {
	followingTeacherService := model.NewFollowingTeacherService(n.db)
	n.lessonService = model.NewLessonService(n.db)

	teacherIDs, err := followingTeacherService.FindTeacherIDsByUserID(user.ID)
	if err != nil {
		return errors.Wrapperf(err, "Failed to FindTeacherIDsByUserID(): userID=%v", user.ID)
	}

	availableLessonsPerTeacher := make(map[uint32][]*model.Lesson, 1000)
	allFetchedLessons := make([]*model.Lesson, 0, 5000)
	for _, teacherID := range teacherIDs {
		teacher, fetchedLessons, newAvailableLessons, err := n.fetchAndExtractNewAvailableLessons(teacherID)
		if err != nil {
			switch err.(type) {
			case *errors.NotFound:
				// TODO: update teacher table flag
				// TODO: Not need to log
				logger.AppLogger.Warn("Cannot fetch teacher", zap.Uint("teacherID", uint(teacherID)))
				continue
			default:
				return err
			}
		}

		allFetchedLessons = append(allFetchedLessons, fetchedLessons...)
		n.teachers[teacherID] = teacher
		if len(newAvailableLessons) > 0 {
			availableLessonsPerTeacher[teacherID] = newAvailableLessons
		}
	}

	if err := n.sendNotificationToUser(user, availableLessonsPerTeacher); err != nil {
		return err
	}

	if !n.dryRun {
		n.lessonService.UpdateLessons(allFetchedLessons)
	}

	time.Sleep(500 * time.Millisecond)

	return nil
}

// Returns teacher, fetchedLessons, newAvailableLessons, error
func (n *Notifier) fetchAndExtractNewAvailableLessons(teacherID uint32) (
	*model.Teacher, []*model.Lesson, []*model.Lesson, error,
) {
	teacher, fetchedLessons, err := lessonFetcher.Fetch(teacherID)
	if err != nil {
		logger.AppLogger.Error(
			"TeacherLessonFetcher.Fetch",
			zap.Uint("teacherID", uint(teacherID)), zap.Error(err),
		)
		return nil, nil, nil, err
	}
	logger.AppLogger.Info(
		"TeacherLessonFetcher.Fetch",
		zap.Uint("teacherID", uint(teacher.ID)),
		zap.String("teacherName", teacher.Name),
		zap.Int("fetchedLessons", len(fetchedLessons)),
	)

	//fmt.Printf("fetchedLessons ---\n")
	//for _, l := range fetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	now := time.Now()
	fromDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, config.LocalTimezone())
	toDate := fromDate.Add(24 * 6 * time.Hour)
	lastFetchedLessons, err := n.lessonService.FindLessons(teacher.ID, fromDate, toDate)
	if err != nil {
		return nil, nil, nil, err
	}
	//fmt.Printf("lastFetchedLessons ---\n")
	//for _, l := range lastFetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	newAvailableLessons := n.lessonService.GetNewAvailableLessons(lastFetchedLessons, fetchedLessons)
	//fmt.Printf("newAvailableLessons ---\n")
	//for _, l := range newAvailableLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}
	return teacher, fetchedLessons, newAvailableLessons, nil
}

func (n *Notifier) sendNotificationToUser(
	user *model.User,
	lessonsPerTeacher map[uint32][]*model.Lesson,
) error {
	lessonsCount := 0
	var teacherIDs []int
	for teacherID, lessons := range lessonsPerTeacher {
		teacherIDs = append(teacherIDs, int(teacherID))
		lessonsCount += len(lessons)
	}
	if lessonsCount == 0 {
		// Don't send notification
		return nil
	}

	sort.Ints(teacherIDs)
	var teacherIDs2 []uint32
	var teacherNames []string
	for _, id := range teacherIDs {
		teacherIDs2 = append(teacherIDs2, uint32(id))
		teacherNames = append(teacherNames, n.teachers[uint32(id)].Name)
	}

	t := template.New("email")
	t = template.Must(t.Parse(getEmailTemplateJP()))
	type TemplateData struct {
		TeacherIDs        []uint32
		Teachers          map[uint32]*model.Teacher
		LessonsPerTeacher map[uint32][]*model.Lesson
		WebURL            string
	}
	data := &TemplateData{
		TeacherIDs:        teacherIDs2,
		Teachers:          n.teachers,
		LessonsPerTeacher: lessonsPerTeacher,
		WebURL:            config.WebURL(),
	}

	var body bytes.Buffer
	if err := t.Execute(&body, data); err != nil {
		return errors.InternalWrapf(err, "Failed to execute template.")
	}
	//fmt.Printf("--- mail ---\n%s", body.String())

	//subject := "Schedule of teacher " + strings.Join(teacherNames, ", ")
	subject := strings.Join(teacherNames, ", ") + "の空きレッスンがあります"
	sender := &EmailNotificationSender{}
	return sender.Send(user, subject, body.String())
}

func getEmailTemplateJP() string {
	return strings.TrimSpace(`
{{- range $teacherID := .TeacherIDs }}
{{- $teacher := index $.Teachers $teacherID -}}
--- {{ $teacher.Name }} ---
  {{- $lessons := index $.LessonsPerTeacher $teacherID }}
  {{- range $lesson := $lessons }}
{{ $lesson.Datetime.Format "2006-01-02 15:04" }}
  {{- end }}

レッスンの予約はこちらから:
PC
Mobile

{{ end }}
空きレッスンの通知の解除はこちら
	`)
}

func getEmailTemplateEN() string {
	return strings.TrimSpace(`
{{- range $teacherID := .TeacherIDs }}
{{- $teacher := index $.Teachers $teacherID -}}
--- {{ $teacher.Name }} ---
  {{- $lessons := index $.LessonsPerTeacher $teacherID }}
  {{- range $lesson := $lessons }}
{{ $lesson.Datetime.Format "2006-01-02 15:04" }}
  {{- end }}

Reserve here:
PC
Mobile
{{ end }}
Click here if you want to stop notification of the teacher.
	`)
}

type NotificationSender interface {
	Send(user *model.User, subject, body string) error
}

type EmailNotificationSender struct{}

func (s *EmailNotificationSender) Send(user *model.User, subject, body string) error {
	from := mail.NewEmail("lekcije", "lekcije@lekcije.com")
	to := mail.NewEmail(user.Name, user.Email.Raw())
	content := mail.NewContent("text/html", strings.Replace(body, "\n", "
", -1))
	m := mail.NewV3MailInit(from, subject, to, content)

	req := sendgrid.GetRequest(
		os.Getenv("SENDGRID_API_KEY"),
		"/v3/mail/send",
		"https://api.sendgrid.com",
	)
	req.Method = "POST"
	req.Body = mail.GetRequestBody(m)
	resp, err := sendgrid.API(req)
	if err != nil {
		return errors.InternalWrapf(err, "Failed to send email by sendgrid")
	}
	if resp.StatusCode >= 300 {
		message := fmt.Sprintf(
			"Failed to send email by sendgrid: statusCode=%v, body=%v",
			resp.StatusCode, strings.Replace(resp.Body, "\n", "\\n", -1),
		)
		logger.AppLogger.Error(message)
		return errors.InternalWrapf(
			err,
			"Failed to send email by sendgrid: statusCode=%v",
			resp.StatusCode,
		)
	}

	return nil
}
<|endoftext|>


<commit_before>package throttler

import "fmt"

type httpPkg struct{}

func (httpPkg) Get(url string) {}

var http httpPkg

// This example fetches several URLs concurrently,
// using a Throttler to block until all the fetches are complete.
// Compare to http://golang.org/pkg/sync/#example_WaitGroup
func ExampleThrottler() {
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	// Create a new Throttler that will get 2 urls at a time
	t := throttler.New(2, len(urls))
	for _, url := range urls {
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Fetch the URL.
			err := http.Get(url)
			// Let Throttler know when the goroutine completes
			// so it can dispatch another worker
			t.Done(err)
		}(url)
		// Pauses until a worker is available or all jobs have been completed
		// Returning the total number of goroutines that have errored
		// lets you choose to break out of the loop without starting any more
		errorCount := t.Throttle()
	}
}

// This example fetches several URLs concurrently,
// using a Throttler to block until all the fetches are complete
// and checks the errors returned.
// Compare to http://golang.org/pkg/sync/#example_WaitGroup
func ExampleThrottler_errors() error {
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	// Create a new Throttler that will get 2 urls at a time
	t := New(2, len(urls))
	for _, url := range urls {
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Let Throttler know when the goroutine completes
			// so it can dispatch another worker
			defer t.Done(nil)
			// Fetch the URL.
			http.Get(url)
		}(url)
		// Pauses until a worker is available or all jobs have been completed
		t.Throttle()
	}

	if t.Err() != nil {
		// Loop through the errors to see the details
		for i, err := range t.Errs() {
			fmt.Printf("error #%d: %s", i, err)
		}
		return t.Err()
	}

	return nil
}
<commit_msg>Fixed example_test.go<commit_after>package throttler

import "fmt"

type httpPkg struct{}

func (httpPkg) Get(url string) error { return nil }

var http httpPkg

// This example fetches several URLs concurrently,
// using a Throttler to block until all the fetches are complete.
// Compare to http://golang.org/pkg/sync/#example_WaitGroup
func ExampleThrottler() {
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	// Create a new Throttler that will get 2 urls at a time
	t := New(2, len(urls))
	for _, url := range urls {
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Fetch the URL.
			err := http.Get(url)
			// Let Throttler know when the goroutine completes
			// so it can dispatch another worker
			t.Done(err)
		}(url)
		// Pauses until a worker is available or all jobs have been completed
		// Returning the total number of goroutines that have errored
		// lets you choose to break out of the loop without starting any more
		errorCount := t.Throttle()
		if errorCount > 0 {
			break
		}
	}
}

// This example fetches several URLs concurrently,
// using a Throttler to block until all the fetches are complete
// and checks the errors returned.
// Compare to http://golang.org/pkg/sync/#example_WaitGroup
func ExampleThrottler_errors() error {
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	// Create a new Throttler that will get 2 urls at a time
	t := New(2, len(urls))
	for _, url := range urls {
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Let Throttler know when the goroutine completes
			// so it can dispatch another worker
			defer t.Done(nil)
			// Fetch the URL.
			http.Get(url)
		}(url)
		// Pauses until a worker is available or all jobs have been completed
		t.Throttle()
	}

	if t.Err() != nil {
		// Loop through the errors to see the details
		for i, err := range t.Errs() {
			fmt.Printf("error #%d: %s", i, err)
		}
		return t.Err()
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright...

// This example demonstrates opening a Connection and doing some basic operations.
package swift_test

import (
	"fmt"

	"github.com/ncw/swift"
)

func ExampleConnection() {
	// Create a v1 auth connection
	c := swift.Connection{
		// This should be your username
		UserName: "user",
		// This should be your api key
		ApiKey: "key",
		// This should be a v1 auth url, eg
		//  Rackspace US        https://auth.api.rackspacecloud.com/v1.0
		//  Rackspace UK        https://lon.auth.api.rackspacecloud.com/v1.0
		//  Memset Memstore UK  https://auth.storage.memset.com/v1.0
		AuthUrl: "auth_url",
	}

	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}
	// List all the containers
	containers, err := c.ContainerNames(nil)
	fmt.Println(containers)
	// etc...

	// ------ or alternatively create a v2 connection ------

	// Create a v2 auth connection
	c = swift.Connection{
		// This is the sub user for the storage - eg "admin"
		UserName: "user",
		// This should be your api key
		ApiKey: "key",
		// This should be a version2 auth url, eg
		//  Rackspace v2        https://identity.api.rackspacecloud.com/v2.0
		//  Memset Memstore v2  https://auth.storage.memset.com/v2.0
		AuthUrl: "v2_auth_url",
		// Region to use - default is use first region if unset
		Region: "LON",
		// Name of the tenant - this is likely your username
		Tenant: "jim",
	}

	// as above...
}

var container string

func ExampleConnection_ObjectsWalk() {
	objects := make([]string, 0)
	err := c.ObjectsWalk(container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
		newObjects, err := c.ObjectNames(container, opts)
		if err == nil {
			objects = append(objects, newObjects...)
		}
		return newObjects, err
	})
	fmt.Println("Found all the objects", objects, err)
}

func ExampleConnection_VersionContainerCreate() {
	// Use the helper method to create the current and versions container.
	if err := c.VersionContainerCreate("cds", "cd-versions"); err != nil {
		fmt.Print(err.Error())
	}
}

func ExampleConnection_VersionEnable() {
	// Build the containers manually and enable them.
	if err := c.ContainerCreate("movie-versions", nil); err != nil {
		fmt.Print(err.Error())
	}
	if err := c.ContainerCreate("movies", nil); err != nil {
		fmt.Print(err.Error())
	}
	if err := c.VersionEnable("movies", "movie-versions"); err != nil {
		fmt.Print(err.Error())
	}

	// Access the primary container as usual with ObjectCreate(), ObjectPut(), etc.
	// etc...
}

func ExampleConnection_VersionDisable() {
	// Disable versioning on a container.  Note that this does not delete the versioning container.
	c.VersionDisable("movies")
}
<commit_msg>Fix go vet errors<commit_after>// Copyright...

// This example demonstrates opening a Connection and doing some basic operations.
package swift_test

import (
	"fmt"

	"github.com/ncw/swift"
)

func ExampleConnection() {
	// Create a v1 auth connection
	c := &swift.Connection{
		// This should be your username
		UserName: "user",
		// This should be your api key
		ApiKey: "key",
		// This should be a v1 auth url, eg
		//  Rackspace US        https://auth.api.rackspacecloud.com/v1.0
		//  Rackspace UK        https://lon.auth.api.rackspacecloud.com/v1.0
		//  Memset Memstore UK  https://auth.storage.memset.com/v1.0
		AuthUrl: "auth_url",
	}

	// Authenticate
	err := c.Authenticate()
	if err != nil {
		panic(err)
	}
	// List all the containers
	containers, err := c.ContainerNames(nil)
	fmt.Println(containers)
	// etc...

	// ------ or alternatively create a v2 connection ------

	// Create a v2 auth connection
	c = &swift.Connection{
		// This is the sub user for the storage - eg "admin"
		UserName: "user",
		// This should be your api key
		ApiKey: "key",
		// This should be a version2 auth url, eg
		//  Rackspace v2        https://identity.api.rackspacecloud.com/v2.0
		//  Memset Memstore v2  https://auth.storage.memset.com/v2.0
		AuthUrl: "v2_auth_url",
		// Region to use - default is use first region if unset
		Region: "LON",
		// Name of the tenant - this is likely your username
		Tenant: "jim",
	}

	// as above...
}

var container string

func ExampleConnection_ObjectsWalk() {
	objects := make([]string, 0)
	err := c.ObjectsWalk(container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
		newObjects, err := c.ObjectNames(container, opts)
		if err == nil {
			objects = append(objects, newObjects...)
		}
		return newObjects, err
	})
	fmt.Println("Found all the objects", objects, err)
}

func ExampleConnection_VersionContainerCreate() {
	// Use the helper method to create the current and versions container.
	if err := c.VersionContainerCreate("cds", "cd-versions"); err != nil {
		fmt.Print(err.Error())
	}
}

func ExampleConnection_VersionEnable() {
	// Build the containers manually and enable them.
	if err := c.ContainerCreate("movie-versions", nil); err != nil {
		fmt.Print(err.Error())
	}
	if err := c.ContainerCreate("movies", nil); err != nil {
		fmt.Print(err.Error())
	}
	if err := c.VersionEnable("movies", "movie-versions"); err != nil {
		fmt.Print(err.Error())
	}

	// Access the primary container as usual with ObjectCreate(), ObjectPut(), etc.
	// etc...
}

func ExampleConnection_VersionDisable() {
	// Disable versioning on a container.  Note that this does not delete the versioning container.
	c.VersionDisable("movies")
}
<|endoftext|>


<commit_before>package graph_test

import (
	"fmt"
	"math/rand"

	"github.com/urandom/graph"
	"github.com/urandom/graph/base"
)

type Processor interface {
	Process(wd graph.WalkData, output chan<- int)
	Result() int
}

type RandomNumberNode struct {
	graph.Node
	result int
}

type MultiplyNode struct {
	graph.Node
	result int
}

type SummingNode struct {
	graph.Node
	result int
}

func (n *RandomNumberNode) Process(wd graph.WalkData, output chan<- int) {
	n.result = rand.Intn(50-10) + 10

	wd.Close()
}

func (n RandomNumberNode) Result() int {
	return n.result
}

func (n *MultiplyNode) Process(wd graph.WalkData, output chan<- int) {
	parent := wd.Parents[0]

	if p, ok := parent.Node.(Processor); ok {
		n.result = p.Result()*rand.Intn(10-1) + 1
	}

	wd.Close()
}

func (n MultiplyNode) Result() int {
	return n.result
}

func (n *SummingNode) Process(wd graph.WalkData, output chan<- int) {
	for _, parent := range wd.Parents {
		if p, ok := parent.Node.(Processor); ok {
			n.result += p.Result()
		}
	}

	wd.Close()
	output <- n.result
}

func (n SummingNode) Result() int {
	return n.result
}

func Example() {
	root := CreateGraph()

	walker := graph.NewWalker(root)
	data := walker.Walk()

	output := make(chan int)

	for wd := range data {
		if p, ok := wd.Node.(Processor); ok {
			go p.Process(wd, output)
		}
	}

	select {
	case r := <-output:
		fmt.Println(r)
	}
}

func CreateGraph() graph.Linker {
	linkers := make([]graph.Linker, 4)

	for i := range linkers {
		l := base.NewLinker()

		switch i {
		case 0:
			l.Data = &RandomNumberNode{Node: l.Data}
		case 1:
			l.Data = &RandomNumberNode{Node: l.Data}
		case 2:
			l.Data = &MultiplyNode{Node: l.Data}
			l.Connect(linkers[0], l.Connector(graph.InputName), linkers[0].Connector(graph.OutputName, graph.OutputType))
		case 3:
			c := base.NewInputConnector("aux")
			l.InputConnectors[c.Name()] = c
			l.Data = &SummingNode{Node: l.Data}

			l.Connect(linkers[1], l.Connector(graph.InputName), linkers[1].Connector(graph.OutputName, graph.OutputType))
			l.Connect(linkers[2], l.Connector(c.Name()), linkers[2].Connector(graph.OutputName, graph.OutputType))
		}

		linkers[i] = l
	}

	return linkers[0]
}
<commit_msg>always close the walk data<commit_after>package graph_test

import (
	"fmt"
	"math/rand"

	"github.com/urandom/graph"
	"github.com/urandom/graph/base"
)

type Processor interface {
	Process(wd graph.WalkData, output chan<- int)
	Result() int
}

type RandomNumberNode struct {
	graph.Node
	result int
}

type MultiplyNode struct {
	graph.Node
	result int
}

type SummingNode struct {
	graph.Node
	result int
}

func (n *RandomNumberNode) Process(wd graph.WalkData, output chan<- int) {
	n.result = rand.Intn(50-10) + 10

	wd.Close()
}

func (n RandomNumberNode) Result() int {
	return n.result
}

func (n *MultiplyNode) Process(wd graph.WalkData, output chan<- int) {
	parent := wd.Parents[0]

	if p, ok := parent.Node.(Processor); ok {
		n.result = p.Result()*rand.Intn(10-1) + 1
	}

	wd.Close()
}

func (n MultiplyNode) Result() int {
	return n.result
}

func (n *SummingNode) Process(wd graph.WalkData, output chan<- int) {
	for _, parent := range wd.Parents {
		if p, ok := parent.Node.(Processor); ok {
			n.result += p.Result()
		}
	}

	wd.Close()
	output <- n.result
}

func (n SummingNode) Result() int {
	return n.result
}

func Example() {
	root := CreateGraph()

	walker := graph.NewWalker(root)
	data := walker.Walk()

	output := make(chan int)

	for wd := range data {
		if p, ok := wd.Node.(Processor); ok {
			go p.Process(wd, output)
		} else {
			wd.Close()
		}
	}

	select {
	case r := <-output:
		fmt.Println(r)
	}
}

func CreateGraph() graph.Linker {
	linkers := make([]graph.Linker, 4)

	for i := range linkers {
		l := base.NewLinker()

		switch i {
		case 0:
			l.Data = &RandomNumberNode{Node: l.Data}
		case 1:
			l.Data = &RandomNumberNode{Node: l.Data}
		case 2:
			l.Data = &MultiplyNode{Node: l.Data}
			l.Connect(linkers[0], l.Connector(graph.InputName), linkers[0].Connector(graph.OutputName, graph.OutputType))
		case 3:
			c := base.NewInputConnector("aux")
			l.InputConnectors[c.Name()] = c
			l.Data = &SummingNode{Node: l.Data}

			l.Connect(linkers[1], l.Connector(graph.InputName), linkers[1].Connector(graph.OutputName, graph.OutputType))
			l.Connect(linkers[2], l.Connector(c.Name()), linkers[2].Connector(graph.OutputName, graph.OutputType))
		}

		linkers[i] = l
	}

	return linkers[0]
}
<|endoftext|>


<commit_before>package durafmt

import (
	"fmt"
	"math"
	"time"
)

func ExampleParseString() {
	duration, err := ParseString("354h22m3.24s")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(duration) // 2 weeks 18 hours 22 minutes 3 seconds
	// duration.String() // String representation. "2 weeks 18 hours 22 minutes 3 seconds"
}

func ExampleParseString_sequence() {
	for hours := 1.0; hours < 12.0; hours++ {
		hour := fmt.Sprintf("%fh", math.Pow(2, hours))
		duration, err := ParseString(hour)
		if err != nil {
			fmt.Println(err)
		}
		fmt.Println(duration) // 2 hours, 4 hours, ...
	}
}

// Version of durafmt.ParseString() that only returns the first part of the duration string.
func ExampleParseStringShort() {
	duration, err := ParseStringShort("354h22m3.24s")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(duration) // 2 weeks 18 hours 22 minutes 3 seconds
	// duration.String() // String representation. "2 weeks 18 hours 22 minutes 3 seconds"
}

func ExampleParse() {
	timeduration := (354 * time.Hour) + (22 * time.Minute) + (3 * time.Second)
	duration := Parse(timeduration).String()
	fmt.Println(duration) // 2 weeks 18 hours 22 minutes 3 seconds
}

// Version of durafmt.Parse() that only returns the first part of the duration string.
func ExampleParseShort() {
	timeduration := (354 * time.Hour) + (22 * time.Minute) + (3 * time.Second)
	duration := ParseShort(timeduration).String()
	fmt.Println(duration) // 2 weeks
}<commit_msg>fmt examples (again)<commit_after>package durafmt

import (
	"fmt"
	"math"
	"time"
)

func ExampleParseString() {
	duration, err := ParseString("354h22m3.24s")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(duration) // 2 weeks 18 hours 22 minutes 3 seconds
	// duration.String() // String representation. "2 weeks 18 hours 22 minutes 3 seconds"
}

func ExampleParseString_sequence() {
	for hours := 1.0; hours < 12.0; hours++ {
		hour := fmt.Sprintf("%fh", math.Pow(2, hours))
		duration, err := ParseString(hour)
		if err != nil {
			fmt.Println(err)
		}
		fmt.Println(duration) // 2 hours, 4 hours, ...
	}
}

// Version of durafmt.ParseString() that only returns the first part of the duration string.
func ExampleParseStringShort() {
	duration, err := ParseStringShort("354h22m3.24s")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(duration) // 2 weeks 18 hours 22 minutes 3 seconds
	// duration.String() // String representation. "2 weeks 18 hours 22 minutes 3 seconds"
}

func ExampleParse() {
	timeduration := (354 * time.Hour) + (22 * time.Minute) + (3 * time.Second)
	duration := Parse(timeduration).String()
	fmt.Println(duration) // 2 weeks 18 hours 22 minutes 3 seconds
}

// Version of durafmt.Parse() that only returns the first part of the duration string.
func ExampleParseShort() {
	timeduration := (354 * time.Hour) + (22 * time.Minute) + (3 * time.Second)
	duration := ParseShort(timeduration).String()
	fmt.Println(duration) // 2 weeks
}
<|endoftext|>


<commit_before>package socks5_test

import (
	"io"
	"log"
	"net"
	"time"

	"github.com/txthinking/socks5"
)

func ExampleSocks5Server() {
	timeout := 60       // 60s
	socks5.Debug = true // enable socks5 debug log

	l, err := net.Listen("tcp", ":1980")
	if err != nil {
		log.Println(err)
		return
	}
	defer l.Close()

	for {
		c, err := l.Accept()
		if err != nil {
			log.Println(err)
			return
		}

		go func(c net.Conn) {
			defer c.Close()
			if err := c.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
				log.Println("set local timeout:", err)
				return
			}

			s5s := &socks5.Server{
				C: c,
				SelectMethod: func(methods []byte) (method byte, got bool) {
					for _, m := range methods {
						if m == socks5.MethodNone {
							method = socks5.MethodNone
							got = true
							return
						}
					}
					return
				},
				SupportedCommands: []byte{socks5.CmdConnect},
			}
			if err := s5s.Negotiate(); err != nil {
				log.Println(err)
				return
			}
			r, err := s5s.GetRequest()
			if err != nil {
				log.Println(err)
				return
			}
			rc, err := r.Connect(c)
			if err != nil {
				log.Println(err)
				return
			}
			defer rc.Close()
			if err := rc.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
				log.Println("set remote timeout:", err)
				return
			}
			go func() {
				_, _ = io.Copy(c, rc)
			}()
			_, _ = io.Copy(rc, c)

		}(c)

	}
}
<commit_msg>fixed misspelling<commit_after>package socks5_test

import (
	"io"
	"log"
	"net"
	"time"

	"github.com/txthinking/socks5"
)

func ExampleServer() {
	timeout := 60       // 60s
	socks5.Debug = true // enable socks5 debug log

	l, err := net.Listen("tcp", ":1980")
	if err != nil {
		log.Println(err)
		return
	}
	defer l.Close()

	for {
		c, err := l.Accept()
		if err != nil {
			log.Println(err)
			return
		}

		go func(c net.Conn) {
			defer c.Close()
			if err := c.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
				log.Println("set local timeout:", err)
				return
			}

			s5s := &socks5.Server{
				C: c,
				SelectMethod: func(methods []byte) (method byte, got bool) {
					for _, m := range methods {
						if m == socks5.MethodNone {
							method = socks5.MethodNone
							got = true
							return
						}
					}
					return
				},
				SupportedCommands: []byte{socks5.CmdConnect},
			}
			if err := s5s.Negotiate(); err != nil {
				log.Println(err)
				return
			}
			r, err := s5s.GetRequest()
			if err != nil {
				log.Println(err)
				return
			}
			rc, err := r.Connect(c)
			if err != nil {
				log.Println(err)
				return
			}
			defer rc.Close()
			if err := rc.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
				log.Println("set remote timeout:", err)
				return
			}
			go func() {
				_, _ = io.Copy(c, rc)
			}()
			_, _ = io.Copy(rc, c)

		}(c)

	}
}
<|endoftext|>


<commit_before>package komorebi

import (
	"strings"
)

type User struct {
	DbModel
	ImagePath string `json:"image_path"`
}

type Users []User

func NewUser(name string, image_path string) User {
	return User{
		ImagePath: image_path,
		DbModel: DbModel{
			Name: name,
		},
	}
}

func (u User) Save() bool {
	return dbMapper.Save(&u)
}

func (u User) TableName() string {
	return "users"
}

func (u Users) TableName() string {
	return "users"
}

func (u User) Destroy() bool {
	if u.Id == 0 {
		return true
	}

	if _, errDelete := dbMapper.Connection.Delete(&u); errDelete != nil {
		Logger.Printf("delete of user failed.", errDelete)
		return false
	}
	return true
}

func (u User) Validate() (bool, map[string][]string) {
	success := true
	errors := map[string][]string{}

	if len(u.Name) <= 0 {
		Logger.Printf("User validation failed. Name not present")
		success = false
		errors["name"] = append(errors["name"], "Name not present.")
	}

	var otherUser User
	GetByName(&otherUser, u.Name)
	if otherUser.Id != 0 && otherUser.Id != u.Id {
		Logger.Printf("User validation failed. Name not uniq")
		success = false
		errors["name"] = append(errors["name"], "Name not uniq.")
	}

	return success, errors
}

func GetUsersByBoardId(board_id int) Users {
	var users Users
	var ids []string

	_, err := dbMapper.Connection.Select(&ids,
		"select UserId from board_users where BoardId=?", board_id)
	if err != nil {
		Logger.Printf("Could not get user_ids by board id", board_id)
	}
	user_ids := strings.Join(ids, ", ")

	_, err = dbMapper.Connection.Select(&users,
		"select * from users where Id IN ("+user_ids+")")
	if err != nil {
		Logger.Printf("Could not get users by board id", board_id)
	}

	return users
}

func GetUsersByTaskId(task_id int) Users {
	var users Users
	var ids []string

	_, err := dbMapper.Connection.Select(&ids,
		"select UserId from task_users where TaskId=?", task_id)
	if err != nil {
		Logger.Println("Could not get user_ids by task id", task_id)
	}
	user_ids := strings.Join(ids, ", ")

	_, err = dbMapper.Connection.Select(&users,
		"select * from users where Id IN ("+user_ids+")")
	if err != nil {
		Logger.Println("Could not get users by task id", task_id)
	}

	return users
}
<commit_msg>optimize sql query<commit_after>package komorebi

import (
	"strings"
)

type User struct {
	DbModel
	ImagePath string `json:"image_path"`
}

type Users []User

func NewUser(name string, image_path string) User {
	return User{
		ImagePath: image_path,
		DbModel: DbModel{
			Name: name,
		},
	}
}

func (u User) Save() bool {
	return dbMapper.Save(&u)
}

func (u User) TableName() string {
	return "users"
}

func (u Users) TableName() string {
	return "users"
}

func (u User) Destroy() bool {
	if u.Id == 0 {
		return true
	}

	if _, errDelete := dbMapper.Connection.Delete(&u); errDelete != nil {
		Logger.Printf("delete of user failed.", errDelete)
		return false
	}
	return true
}

func (u User) Validate() (bool, map[string][]string) {
	success := true
	errors := map[string][]string{}

	if len(u.Name) <= 0 {
		Logger.Printf("User validation failed. Name not present")
		success = false
		errors["name"] = append(errors["name"], "Name not present.")
	}

	var otherUser User
	GetByName(&otherUser, u.Name)
	if otherUser.Id != 0 && otherUser.Id != u.Id {
		Logger.Printf("User validation failed. Name not uniq")
		success = false
		errors["name"] = append(errors["name"], "Name not uniq.")
	}

	return success, errors
}

func GetUsersByBoardId(board_id int) Users {
	users := make([]User, 0)
	var ids []string

	_, err := dbMapper.Connection.Select(&ids,
		"select UserId from board_users where BoardId=?", board_id)
	if err != nil {
		Logger.Printf("Could not get user_ids by board id", board_id)
	}
	user_ids := strings.Join(ids, ", ")

	if len(user_ids) <= 0 {
		return users
	}
	_, err = dbMapper.Connection.Select(&users,
		"select * from users where Id IN ("+user_ids+")")
	if err != nil {
		Logger.Printf("Could not get users by board id", board_id)
	}

	return users
}

func GetUsersByTaskId(task_id int) Users {
	users := make([]User, 0)
	var ids []string

	_, err := dbMapper.Connection.Select(&ids,
		"select UserId from task_users where TaskId=?", task_id)
	if err != nil {
		Logger.Println("Could not get user_ids by task id", task_id)
	}
	user_ids := strings.Join(ids, ", ")

	if len(user_ids) <= 0 {
		return users
	}

	_, err = dbMapper.Connection.Select(&users,
		"select * from users where Id IN ("+user_ids+")")
	if err != nil {
		Logger.Println("Could not get users by task id", task_id)
	}

	return users
}
<|endoftext|>


<commit_before>package complete_test

import "github.com/posener/complete"

func main() {

	// create a Command object, that represents the command we want
	// to complete.
	run := complete.Command{

		// Name must be exactly as the binary that we want to complete
		Name: "run",

		// Sub defines a list of sub commands of the program,
		// this is recursive, since every command is of type command also.
		Sub: complete.Commands{

			// add a build sub command
			"build": complete.Command{

				// define flags of the build sub command
				Flags: complete.Flags{
					// build sub command has a flag '-fast', which
					// does not expects anything after it.
					"-fast": complete.PredictNothing,
				},
			},
		},

		// define flags of the 'run' main command
		Flags: complete.Flags{

			// a flag '-h' which does not expects anything after it
			"-h": complete.PredictNothing,

			// a flag -o, which expects a file ending with .out after
			// it, the tab completion will auto complete for files matching
			// the given pattern.
			"-o": complete.PredictFiles("*.out"),
		},
	}

	// run the command completion, as part of the main() function.
	// this triggers the autocompletion when needed.
	complete.Run(run)
}
<commit_msg>Remove example test<commit_after><|endoftext|>