<commit_before>//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/opencontainers/specs"
)

type IsolatorCapSet struct {
	Sets []string `json:"set"`
}

// Entry point of oci2aci,
// First convert oci layout to aci layout, then build aci layout to image.
func runOCI2ACI(path string, flagDebug bool) error {
	if flagDebug {
		InitDebug()
	}
	if bValidate := validateOCIProc(path); bValidate != true {
		log.Printf("Conversion stop.")
		return nil
	}

	dirWork := createWorkDir()
	// First, convert layout
	err := convertLayout(path, dirWork)
	if err != nil {
		if debugEnabled {
			log.Printf("Conversion from oci to aci layout failed: %v", err)
		}

	}
	// Second, build image
	err = buildACI(dirWork)
	if err != nil {
		if debugEnabled {
			log.Printf("Generate aci image failed!")
		}
	} else {
		if debugEnabled {
			log.Printf("aci image generated successfully.")
		}
	}

	return nil
}

// Create work directory for the conversion output
func createWorkDir() string {
	idir, err := ioutil.TempDir("", "oci2aci")
	if err != nil {
		return ""
	}
	rootfs := filepath.Join(idir, "rootfs")
	os.MkdirAll(rootfs, 0755)

	data := []byte{}
	if err := ioutil.WriteFile(filepath.Join(idir, "manifest"), data, 0644); err != nil {
		return ""
	}
	return idir
}

// The structure of appc manifest:
// 1.acKind
// 2. acVersion
// 3. name
// 4. labels
//	4.1 version
//	4.2 os
//	4.3 arch
// 5. app
//	5.1 exec
//	5.2 user
//	5.3 group
//	5.4 eventHandlers
//	5.5 workingDirectory
//	5.6 environment
//	5.7 mountPoints
//	5.8 ports
//      5.9 isolators
// 6. annotations
//	6.1 created
//	6.2 authors
//	6.3 homepage
//	6.4 documentation
// 7. dependencies
//	7.1 imageName
//	7.2 imageID
//	7.3 labels
//	7.4 size
// 8. pathWhitelist

func genManifest(path string) *schema.ImageManifest {
	// Get runtime.json and config.json
	runtimePath := path + "/runtime.json"
	configPath := path + "/config.json"

	runtime, err := ioutil.ReadFile(runtimePath)
	if err != nil {
		return nil
	}

	config, err := ioutil.ReadFile(configPath)
	if err != nil {
		return nil
	}

	var spec specs.LinuxSpec
	err = json.Unmarshal(config, &spec)
	if err != nil {
		return nil
	}

	var runSpec specs.RuntimeSpec
	err = json.Unmarshal(runtime, &runSpec)
	if err != nil {
		return nil
	}

	// Begin to convert runtime.json/config.json to manifest
	m := new(schema.ImageManifest)

	// 1. Assemble "acKind" field
	m.ACKind = "ImageManifest"

	// 2. Assemble "acVersion" field
	m.ACVersion = schema.AppContainerVersion

	// 3. Assemble "name" field
	m.Name = "example"

	// 4. Assemble "labels" field
	// 4.1 "version"
	label := new(types.Label)
	label.Name = types.ACIdentifier("version")
	label.Value = spec.Version
	m.Labels = append(m.Labels, *label)
	// 4.2 "os"
	label = new(types.Label)
	label.Name = types.ACIdentifier("os")
	label.Value = spec.Platform.OS
	m.Labels = append(m.Labels, *label)
	// 4.3 "arch"
	label = new(types.Label)
	label.Name = types.ACIdentifier("arch")
	label.Value = spec.Platform.Arch
	m.Labels = append(m.Labels, *label)

	// 5. Assemble "app" field
	app := new(types.App)
	// 5.1 "exec"
	app.Exec = spec.Process.Args
	// 5.2 "user"
	app.User = fmt.Sprintf("%d", spec.Process.User.UID)
	// 5.3 "group"
	app.Group = fmt.Sprintf("%d", spec.Process.User.GID)
	// 5.4 "eventHandlers"
	event := new(types.EventHandler)
	event.Name = "pre-start"
	for index := range runSpec.Hooks.Prestart {
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Env...)
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	event = new(types.EventHandler)
	event.Name = "post-stop"
	for index := range runSpec.Hooks.Poststop {
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Env...)
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	// 5.5 "workingDirectory"
	app.WorkingDirectory = spec.Process.Cwd
	// 5.6 "environment"
	env := new(types.EnvironmentVariable)
	for index := range spec.Process.Env {
		s := strings.Split(spec.Process.Env[index], "=")
		env.Name = s[0]
		env.Value = s[1]
		app.Environment = append(app.Environment, *env)
	}

	// 5.7 "mountPoints"
	for index := range spec.Mounts {
		mount := new(types.MountPoint)
		mount.Name = types.ACName(spec.Mounts[index].Name)
		mount.Path = spec.Mounts[index].Path
		app.MountPoints = append(app.MountPoints, *mount)
	}

	// 5.8 "ports"

	// 5.9 "isolators"
	if len(spec.Linux.Capabilities) != 0 {
		isolatorCapSet := new(IsolatorCapSet)
		isolatorCapSet.Sets = append(isolatorCapSet.Sets, spec.Linux.Capabilities...)

		isolator := new(types.Isolator)
		isolator.Name = types.ACIdentifier(types.LinuxCapabilitiesRetainSetName)
		bytes, _ := json.Marshal(isolatorCapSet)

		valueRaw := json.RawMessage(bytes)
		isolator.ValueRaw = &valueRaw

		app.Isolators = append(app.Isolators, *isolator)
	}

	// 6. "annotations"

	// 7. "dependencies"

	// 8. "pathWhitelist"

	m.App = app

	return m
}

// Convert OCI layout to ACI layout
func convertLayout(srcPath, dstPath string) error {
	src, _ := filepath.Abs(srcPath)
	src += "/rootfs"
	if err := run(exec.Command("cp", "-rf", src, dstPath)); err != nil {
		return err
	}

	m := genManifest(srcPath)

	bytes, err := json.Marshal(m)
	if err != nil {
		return err
	}

	manifestPath := dstPath + "/manifest"

	ioutil.WriteFile(manifestPath, bytes, 0644)
	return nil
}
<commit_msg>use LinuxRuntimeSpec<commit_after>//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/huawei-openlab/oci2aci/Godeps/_workspace/src/github.com/appc/spec/schema"
	"github.com/huawei-openlab/oci2aci/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/huawei-openlab/oci2aci/Godeps/_workspace/src/github.com/opencontainers/specs"
)

type IsolatorCapSet struct {
	Sets []string `json:"set"`
}

// Entry point of oci2aci,
// First convert oci layout to aci layout, then build aci layout to image.
func runOCI2ACI(path string, flagDebug bool) error {
	if flagDebug {
		InitDebug()
	}
	if bValidate := validateOCIProc(path); bValidate != true {
		log.Printf("Conversion stop.")
		return nil
	}

	dirWork := createWorkDir()
	// First, convert layout
	err := convertLayout(path, dirWork)
	if err != nil {
		if debugEnabled {
			log.Printf("Conversion from oci to aci layout failed: %v", err)
		}

	}
	// Second, build image
	err = buildACI(dirWork)
	if err != nil {
		if debugEnabled {
			log.Printf("Generate aci image failed!")
		}
	} else {
		if debugEnabled {
			log.Printf("aci image generated successfully.")
		}
	}

	return nil
}

// Create work directory for the conversion output
func createWorkDir() string {
	idir, err := ioutil.TempDir("", "oci2aci")
	if err != nil {
		return ""
	}
	rootfs := filepath.Join(idir, "rootfs")
	os.MkdirAll(rootfs, 0755)

	data := []byte{}
	if err := ioutil.WriteFile(filepath.Join(idir, "manifest"), data, 0644); err != nil {
		return ""
	}
	return idir
}

// The structure of appc manifest:
// 1.acKind
// 2. acVersion
// 3. name
// 4. labels
//	4.1 version
//	4.2 os
//	4.3 arch
// 5. app
//	5.1 exec
//	5.2 user
//	5.3 group
//	5.4 eventHandlers
//	5.5 workingDirectory
//	5.6 environment
//	5.7 mountPoints
//	5.8 ports
//      5.9 isolators
// 6. annotations
//	6.1 created
//	6.2 authors
//	6.3 homepage
//	6.4 documentation
// 7. dependencies
//	7.1 imageName
//	7.2 imageID
//	7.3 labels
//	7.4 size
// 8. pathWhitelist

func genManifest(path string) *schema.ImageManifest {
	// Get runtime.json and config.json
	runtimePath := path + "/runtime.json"
	configPath := path + "/config.json"

	runtime, err := ioutil.ReadFile(runtimePath)
	if err != nil {
		return nil
	}

	config, err := ioutil.ReadFile(configPath)
	if err != nil {
		return nil
	}

	var spec specs.LinuxSpec
	err = json.Unmarshal(config, &spec)
	if err != nil {
		return nil
	}

	var runSpec specs.LinuxRuntimeSpec
	err = json.Unmarshal(runtime, &runSpec)
	if err != nil {
		return nil
	}

	// Begin to convert runtime.json/config.json to manifest
	m := new(schema.ImageManifest)

	// 1. Assemble "acKind" field
	m.ACKind = "ImageManifest"

	// 2. Assemble "acVersion" field
	m.ACVersion = schema.AppContainerVersion

	// 3. Assemble "name" field
	m.Name = "example"

	// 4. Assemble "labels" field
	// 4.1 "version"
	label := new(types.Label)
	label.Name = types.ACIdentifier("version")
	label.Value = spec.Version
	m.Labels = append(m.Labels, *label)
	// 4.2 "os"
	label = new(types.Label)
	label.Name = types.ACIdentifier("os")
	label.Value = spec.Platform.OS
	m.Labels = append(m.Labels, *label)
	// 4.3 "arch"
	label = new(types.Label)
	label.Name = types.ACIdentifier("arch")
	label.Value = spec.Platform.Arch
	m.Labels = append(m.Labels, *label)

	// 5. Assemble "app" field
	app := new(types.App)
	// 5.1 "exec"
	app.Exec = spec.Process.Args
	// 5.2 "user"
	app.User = fmt.Sprintf("%d", spec.Process.User.UID)
	// 5.3 "group"
	app.Group = fmt.Sprintf("%d", spec.Process.User.GID)
	// 5.4 "eventHandlers"
	event := new(types.EventHandler)
	event.Name = "pre-start"
	for index := range runSpec.Hooks.Prestart {
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Env...)
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	event = new(types.EventHandler)
	event.Name = "post-stop"
	for index := range runSpec.Hooks.Poststop {
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Env...)
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	// 5.5 "workingDirectory"
	app.WorkingDirectory = spec.Process.Cwd
	// 5.6 "environment"
	env := new(types.EnvironmentVariable)
	for index := range spec.Process.Env {
		s := strings.Split(spec.Process.Env[index], "=")
		env.Name = s[0]
		env.Value = s[1]
		app.Environment = append(app.Environment, *env)
	}

	// 5.7 "mountPoints"
	for index := range spec.Mounts {
		mount := new(types.MountPoint)
		mount.Name = types.ACName(spec.Mounts[index].Name)
		mount.Path = spec.Mounts[index].Path
		app.MountPoints = append(app.MountPoints, *mount)
	}

	// 5.8 "ports"

	// 5.9 "isolators"
	if runSpec.Linux.Resources != nil {

	}
	if len(spec.Linux.Capabilities) != 0 {
		isolatorCapSet := new(IsolatorCapSet)
		isolatorCapSet.Sets = append(isolatorCapSet.Sets, spec.Linux.Capabilities...)

		isolator := new(types.Isolator)
		isolator.Name = types.ACIdentifier(types.LinuxCapabilitiesRetainSetName)
		bytes, _ := json.Marshal(isolatorCapSet)

		valueRaw := json.RawMessage(bytes)
		isolator.ValueRaw = &valueRaw

		app.Isolators = append(app.Isolators, *isolator)
	}

	// 6. "annotations"

	// 7. "dependencies"

	// 8. "pathWhitelist"

	m.App = app

	return m
}

// Convert OCI layout to ACI layout
func convertLayout(srcPath, dstPath string) error {
	src, _ := filepath.Abs(srcPath)
	src += "/rootfs"
	if err := run(exec.Command("cp", "-rf", src, dstPath)); err != nil {
		return err
	}

	m := genManifest(srcPath)

	bytes, err := json.Marshal(m)
	if err != nil {
		return err
	}

	manifestPath := dstPath + "/manifest"

	ioutil.WriteFile(manifestPath, bytes, 0644)
	return nil
}
<|endoftext|>


<commit_before>package tlc

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/itchio/arkive/zip"
	"github.com/itchio/httpkit/progress"

	"github.com/itchio/wharf/eos"
	"github.com/pkg/errors"
)

const (
	// ModeMask is or'd with files being diffed
	ModeMask = 0644

	// NullPath can be specified instead of a directory to yield an empty container
	NullPath = "/dev/null"
)

var (
	ErrUnrecognizedContainer = errors.New("Unrecognized container: should either be a directory, or a .zip archive")
)

// A FilterFunc allows ignoring certain files or directories when walking the filesystem
// When a directory is ignored by a FilterFunc, all its children are, too!
type FilterFunc func(fileInfo os.FileInfo) bool

// DefaultFilter is a passthrough that filters out no files at all
var DefaultFilter FilterFunc = func(fileInfo os.FileInfo) bool {
	return true
}

type WalkOpts struct {
	// Filter decides which files to exclude from the walk
	Filter FilterFunc

	// Dereference walks symlinks as if they were their targets
	Dereference bool
}

// WalkAny tries to retrieve container information on containerPath. It supports:
// the empty container (/dev/null), local directories, zip archives, or single files
func WalkAny(containerPath string, opts *WalkOpts) (*Container, error) {
	// empty container case
	if containerPath == NullPath {
		return &Container{}, nil
	}

	file, err := eos.Open(containerPath)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return nil, errors.WithStack(err)
	}

	if stat.IsDir() {
		// local directory case
		return WalkDir(containerPath, opts)
	}

	// zip archive case
	if strings.HasSuffix(strings.ToLower(stat.Name()), ".zip") {
		zr, err := zip.NewReader(file, stat.Size())
		if err != nil {
			return nil, errors.WithStack(err)
		}
		return WalkZip(zr, opts)
	}

	// single file case
	return WalkSingle(file)
}

// WalkSingle returns a container with a single file
func WalkSingle(file eos.File) (*Container, error) {
	stats, err := file.Stat()
	if err != nil {
		return nil, err
	}

	if !stats.Mode().IsRegular() {
		return nil, errors.Errorf("%s: not a regular file, can only WalkSingle regular files", stats.Name())
	}

	container := &Container{
		Files: []*File{
			&File{
				Mode:   uint32(stats.Mode()),
				Size:   int64(stats.Size()),
				Offset: 0,
				Path:   filepath.Base(stats.Name()),
			},
		},
		Size: stats.Size(),
	}
	return container, nil
}

// WalkDir retrieves information on all files, directories, and symlinks in a directory
func WalkDir(basePathIn string, opts *WalkOpts) (*Container, error) {
	filter := opts.Filter

	if filter == nil {
		filter = DefaultFilter
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	currentlyWalking := make(map[string]bool)

	TotalOffset := int64(0)

	var makeEntryCallback func(BasePath string, LocationPath string) filepath.WalkFunc

	makeEntryCallback = func(BasePath string, LocationPath string) filepath.WalkFunc {
		return func(FullPath string, fileInfo os.FileInfo, err error) error {
			// we shouldn't encounter any error crawling the repo
			if err != nil {
				if os.IsPermission(err) {
					// ...except permission errors, those are fine
					log.Printf("Permission error: %s\n", err.Error())
				} else {
					return errors.WithStack(err)
				}
			}

			Path, err := filepath.Rel(BasePath, FullPath)
			if err != nil {
				return errors.WithStack(err)
			}

			Path = filepath.Join(LocationPath, Path)

			Path = filepath.ToSlash(Path)
			if Path == "." {
				// Don't store a single folder named "."
				return nil
			}

			// os.Walk does not follow symlinks, so we must do it
			// manually if Dereference is set
			if opts.Dereference && fileInfo.Mode()&os.ModeSymlink > 0 {
				fileInfo, err = os.Stat(FullPath)
				if err != nil {
					return errors.WithStack(err)
				}

				if fileInfo.Mode().IsDir() {
					Dest, err := os.Readlink(FullPath)
					if err != nil {
						return errors.WithStack(err)
					}

					var JoinedDest string
					if filepath.IsAbs(Dest) {
						JoinedDest = Dest
					} else {
						JoinedDest = filepath.Join(filepath.Dir(FullPath), Dest)
					}

					CleanDest := filepath.Clean(JoinedDest)

					if currentlyWalking[CleanDest] {
						err := fmt.Errorf("symlinks recurse onto %s, cowardly refusing to walk infinite container", CleanDest)
						return errors.WithStack(err)
					}

					currentlyWalking[CleanDest] = true
					err = filepath.Walk(CleanDest, makeEntryCallback(CleanDest, Path))
					delete(currentlyWalking, CleanDest)
					if err != nil {
						return errors.WithStack(err)
					}
				}
			}

			// don't end up with files we (the patcher) can't modify
			Mode := fileInfo.Mode() | ModeMask

			if !filter(fileInfo) {
				if Mode.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if Mode.IsDir() {
				Dirs = append(Dirs, &Dir{Path: Path, Mode: uint32(Mode)})
			} else if Mode.IsRegular() {
				Size := fileInfo.Size()
				Offset := TotalOffset
				OffsetEnd := Offset + Size

				Files = append(Files, &File{Path: Path, Mode: uint32(Mode), Size: Size, Offset: Offset})
				TotalOffset = OffsetEnd
			} else if Mode&os.ModeSymlink > 0 {
				Dest, err := os.Readlink(FullPath)
				if err != nil {
					return errors.WithStack(err)
				}

				Dest = filepath.ToSlash(Dest)
				Symlinks = append(Symlinks, &Symlink{Path: Path, Mode: uint32(Mode), Dest: Dest})
			}

			return nil
		}
	}

	if basePathIn == NullPath {
		// empty container is fine - /dev/null is legal even on Win32 where it doesn't exist
	} else {
		basePathIn, err := filepath.Abs(basePathIn)
		if err != nil {
			return nil, errors.WithStack(err)
		}

		fi, err := os.Lstat(basePathIn)
		if err != nil {
			return nil, errors.WithStack(err)
		}

		if !fi.IsDir() {
			return nil, errors.Errorf("can't walk non-directory %s", basePathIn)
		}

		currentlyWalking[basePathIn] = true
		err = filepath.Walk(basePathIn, makeEntryCallback(basePathIn, "."))
		if err != nil {
			return nil, errors.WithStack(err)
		}
	}

	container := &Container{Size: TotalOffset, Dirs: Dirs, Symlinks: Symlinks, Files: Files}
	return container, nil
}

// WalkZip walks all file in a zip archive and returns a container
func WalkZip(zr *zip.Reader, opts *WalkOpts) (*Container, error) {
	filter := opts.Filter

	if filter == nil {
		// default filter is a passthrough
		filter = func(fileInfo os.FileInfo) bool {
			return true
		}
	}

	if opts.Dereference {
		return nil, errors.New("Dereference is not supporting when walking a zip")
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	dirMap := make(map[string]os.FileMode)

	TotalOffset := int64(0)

	for _, file := range zr.File {
		fileName := filepath.ToSlash(filepath.Clean(filepath.ToSlash(file.Name)))

		// don't trust zip files to have directory entries for
		// all directories. it's a miracle anything works.
		dir := path.Dir(fileName)
		if dir != "" && dir != "." && dirMap[dir] == 0 {
			dirMap[dir] = os.FileMode(0755)
		}

		info := file.FileInfo()
		mode := file.Mode() | ModeMask

		if info.IsDir() {
			dirMap[fileName] = mode
		} else if mode&os.ModeSymlink > 0 {
			var linkname []byte

			err := func() error {
				reader, err := file.Open()
				if err != nil {
					return errors.WithStack(err)
				}
				defer reader.Close()

				linkname, err = ioutil.ReadAll(reader)
				if err != nil {
					return errors.WithStack(err)
				}
				return nil
			}()

			if err != nil {
				return nil, errors.WithStack(err)
			}

			Symlinks = append(Symlinks, &Symlink{
				Path: fileName,
				Dest: string(linkname),
				Mode: uint32(mode),
			})
		} else {
			Size := int64(file.UncompressedSize64)

			Files = append(Files, &File{
				Path:   fileName,
				Mode:   uint32(mode),
				Size:   Size,
				Offset: TotalOffset,
			})

			TotalOffset += Size
		}
	}

	for dirPath, dirMode := range dirMap {
		Dirs = append(Dirs, &Dir{
			Path: dirPath,
			Mode: uint32(dirMode),
		})
	}

	container := &Container{
		Size:     TotalOffset,
		Dirs:     Dirs,
		Symlinks: Symlinks,
		Files:    Files,
	}
	return container, nil
}

// Stats return a human-readable summary of the contents of a container
func (container *Container) Stats() string {
	return fmt.Sprintf("%d files, %d dirs, %d symlinks",
		len(container.Files), len(container.Dirs), len(container.Symlinks))
}

var _ fmt.Formatter = (*Container)(nil)

// Stats return a human-readable summary of the contents of a container
func (container *Container) Format(f fmt.State, c rune) {
	fmt.Fprintf(f, "%s (%s)",
		progress.FormatBytes(container.Size),
		container.Stats(),
	)
}

// IsSingleFile returns true if the container contains
// exactly one files, and no directories or symlinks.
func (container *Container) IsSingleFile() bool {
	if len(container.Files) == 1 && len(container.Dirs) == 0 && len(container.Symlinks) == 0 {
		return true
	}
	return false
}
<commit_msg>Introduce 'wrapping', see https://github.com/itchio/itch/issues/2147<commit_after>package tlc

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/itchio/arkive/zip"
	"github.com/itchio/httpkit/progress"

	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/state"
	"github.com/pkg/errors"
)

const (
	// ModeMask is or'd with files being diffed
	ModeMask = 0644

	// NullPath can be specified instead of a directory to yield an empty container
	NullPath = "/dev/null"
)

var (
	ErrUnrecognizedContainer = errors.New("Unrecognized container: should either be a directory, or a .zip archive")
)

// A FilterFunc allows ignoring certain files or directories when walking the filesystem
// When a directory is ignored by a FilterFunc, all its children are, too!
type FilterFunc func(fileInfo os.FileInfo) bool

// DefaultFilter is a passthrough that filters out no files at all
var DefaultFilter FilterFunc = func(fileInfo os.FileInfo) bool {
	return true
}

type WalkOpts struct {
	// "Wrapping" solves the problem where we're walking:
	// /foo/bar/Sample.app
	// But we want all files, dirs and symlinks in the container to start with "Sample.app/".
	//
	// What we do is we adjust the walked path to:
	// /foo/bar
	// And we set `WrappedDir` to `Sample.app`.
	//
	// This is only used by `WalkDir`. It behaves as if the `/foo/bar` directory
	// only contained `Sample.app`, and nothing else.
	WrappedDir string

	// Filter decides which files to exclude from the walk
	Filter FilterFunc

	// Dereference walks symlinks as if they were their targets
	Dereference bool
}

// Wrap the container path if it's a directory, and it ends in .app
func (opts *WalkOpts) AutoWrap(containerPathPtr *string, consumer *state.Consumer) {
	if !opts.normalizeContainerPath(containerPathPtr) {
		return
	}

	// macOS app bundles should be pushed *as if* they were
	// in another folder, so their structure is recreated correctly.
	if strings.HasSuffix(strings.ToLower(*containerPathPtr), ".app") {
		consumer.Infof("(%s) is a macOS app bundle, making it the top-level directory in the container", *containerPathPtr)
		opts.Wrap(containerPathPtr)
	}
}

// Wrap the container path if it's a directory
func (opts *WalkOpts) Wrap(containerPathPtr *string) {
	if !opts.normalizeContainerPath(containerPathPtr) {
		return
	}

	opts.WrappedDir = filepath.Base(*containerPathPtr)
	*containerPathPtr = filepath.Dir(*containerPathPtr)
}

func (opts *WalkOpts) normalizeContainerPath(containerPathPtr *string) bool {
	stats, err := os.Stat(*containerPathPtr)
	if err != nil {
		// might be a remote path, might not exist, might not have permission to read
		// can't do anything, will err later
		return false
	}

	if !stats.IsDir() {
		return false
	}

	absPath, err := filepath.Abs(*containerPathPtr)
	if err != nil {
		// might be a remote path (http://, https://, etc.)
		// can't do anything, will err later
		return false
	}

	*containerPathPtr = absPath
	return true
}

// WalkAny tries to retrieve container information on containerPath. It supports:
// the empty container (/dev/null), local directories, zip archives, or single files
func WalkAny(containerPath string, opts *WalkOpts) (*Container, error) {
	// empty container case
	if containerPath == NullPath {
		return &Container{}, nil
	}

	file, err := eos.Open(containerPath)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return nil, errors.WithStack(err)
	}

	if stat.IsDir() {
		// local directory case
		return WalkDir(containerPath, opts)
	}

	// zip archive case
	if strings.HasSuffix(strings.ToLower(stat.Name()), ".zip") {
		zr, err := zip.NewReader(file, stat.Size())
		if err != nil {
			return nil, errors.WithStack(err)
		}
		return WalkZip(zr, opts)
	}

	// single file case
	return WalkSingle(file)
}

// WalkSingle returns a container with a single file
func WalkSingle(file eos.File) (*Container, error) {
	stats, err := file.Stat()
	if err != nil {
		return nil, err
	}

	if !stats.Mode().IsRegular() {
		return nil, errors.Errorf("%s: not a regular file, can only WalkSingle regular files", stats.Name())
	}

	container := &Container{
		Files: []*File{
			&File{
				Mode:   uint32(stats.Mode()),
				Size:   int64(stats.Size()),
				Offset: 0,
				Path:   filepath.Base(stats.Name()),
			},
		},
		Size: stats.Size(),
	}
	return container, nil
}

// WalkDir retrieves information on all files, directories, and symlinks in a directory
func WalkDir(basePathIn string, opts *WalkOpts) (*Container, error) {
	filter := opts.Filter

	if filter == nil {
		filter = DefaultFilter
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	currentlyWalking := make(map[string]bool)

	TotalOffset := int64(0)

	var makeEntryCallback func(BasePath string, LocationPath string) filepath.WalkFunc

	makeEntryCallback = func(BasePath string, LocationPath string) filepath.WalkFunc {
		return func(FullPath string, fileInfo os.FileInfo, err error) error {
			// we shouldn't encounter any error crawling the repo
			if err != nil {
				if os.IsPermission(err) {
					// ...except permission errors, those are fine
					log.Printf("Permission error: %s\n", err.Error())
				} else {
					return errors.WithStack(err)
				}
			}

			Path, err := filepath.Rel(BasePath, FullPath)
			if err != nil {
				return errors.WithStack(err)
			}

			Path = filepath.Join(LocationPath, Path)

			Path = filepath.ToSlash(Path)
			if Path == "." {
				// Don't store a single folder named "."
				return nil
			}

			// os.Walk does not follow symlinks, so we must do it
			// manually if Dereference is set
			if opts.Dereference && fileInfo.Mode()&os.ModeSymlink > 0 {
				fileInfo, err = os.Stat(FullPath)
				if err != nil {
					return errors.WithStack(err)
				}

				if fileInfo.Mode().IsDir() {
					Dest, err := os.Readlink(FullPath)
					if err != nil {
						return errors.WithStack(err)
					}

					var JoinedDest string
					if filepath.IsAbs(Dest) {
						JoinedDest = Dest
					} else {
						JoinedDest = filepath.Join(filepath.Dir(FullPath), Dest)
					}

					CleanDest := filepath.Clean(JoinedDest)

					if currentlyWalking[CleanDest] {
						err := fmt.Errorf("symlinks recurse onto %s, cowardly refusing to walk infinite container", CleanDest)
						return errors.WithStack(err)
					}

					currentlyWalking[CleanDest] = true
					err = filepath.Walk(CleanDest, makeEntryCallback(CleanDest, Path))
					delete(currentlyWalking, CleanDest)
					if err != nil {
						return errors.WithStack(err)
					}
				}
			}

			// don't end up with files we (the patcher) can't modify
			Mode := fileInfo.Mode() | ModeMask

			if !filter(fileInfo) {
				if Mode.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if Mode.IsDir() {
				Dirs = append(Dirs, &Dir{Path: Path, Mode: uint32(Mode)})
			} else if Mode.IsRegular() {
				Size := fileInfo.Size()
				Offset := TotalOffset
				OffsetEnd := Offset + Size

				Files = append(Files, &File{Path: Path, Mode: uint32(Mode), Size: Size, Offset: Offset})
				TotalOffset = OffsetEnd
			} else if Mode&os.ModeSymlink > 0 {
				Dest, err := os.Readlink(FullPath)
				if err != nil {
					return errors.WithStack(err)
				}

				Dest = filepath.ToSlash(Dest)
				Symlinks = append(Symlinks, &Symlink{Path: Path, Mode: uint32(Mode), Dest: Dest})
			}

			return nil
		}
	}

	if basePathIn == NullPath {
		// empty container is fine - /dev/null is legal even on Win32 where it doesn't exist
	} else {
		basePathIn, err := filepath.Abs(basePathIn)
		if err != nil {
			return nil, errors.WithStack(err)
		}

		fi, err := os.Lstat(basePathIn)
		if err != nil {
			return nil, errors.WithStack(err)
		}

		if !fi.IsDir() {
			return nil, errors.Errorf("can't walk non-directory %s", basePathIn)
		}

		baseName := "."
		if opts.WrappedDir != "" {
			baseName = opts.WrappedDir
			basePathIn = filepath.Join(basePathIn, opts.WrappedDir)
		}

		currentlyWalking[basePathIn] = true
		err = filepath.Walk(basePathIn, makeEntryCallback(basePathIn, baseName))
		if err != nil {
			return nil, errors.WithStack(err)
		}
	}

	container := &Container{Size: TotalOffset, Dirs: Dirs, Symlinks: Symlinks, Files: Files}
	return container, nil
}

// WalkZip walks all file in a zip archive and returns a container
func WalkZip(zr *zip.Reader, opts *WalkOpts) (*Container, error) {
	filter := opts.Filter

	if filter == nil {
		// default filter is a passthrough
		filter = func(fileInfo os.FileInfo) bool {
			return true
		}
	}

	if opts.Dereference {
		return nil, errors.New("Dereference is not supporting when walking a zip")
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	dirMap := make(map[string]os.FileMode)

	TotalOffset := int64(0)

	for _, file := range zr.File {
		fileName := filepath.ToSlash(filepath.Clean(filepath.ToSlash(file.Name)))

		// don't trust zip files to have directory entries for
		// all directories. it's a miracle anything works.
		dir := path.Dir(fileName)
		if dir != "" && dir != "." && dirMap[dir] == 0 {
			dirMap[dir] = os.FileMode(0755)
		}

		info := file.FileInfo()
		mode := file.Mode() | ModeMask

		if info.IsDir() {
			dirMap[fileName] = mode
		} else if mode&os.ModeSymlink > 0 {
			var linkname []byte

			err := func() error {
				reader, err := file.Open()
				if err != nil {
					return errors.WithStack(err)
				}
				defer reader.Close()

				linkname, err = ioutil.ReadAll(reader)
				if err != nil {
					return errors.WithStack(err)
				}
				return nil
			}()

			if err != nil {
				return nil, errors.WithStack(err)
			}

			Symlinks = append(Symlinks, &Symlink{
				Path: fileName,
				Dest: string(linkname),
				Mode: uint32(mode),
			})
		} else {
			Size := int64(file.UncompressedSize64)

			Files = append(Files, &File{
				Path:   fileName,
				Mode:   uint32(mode),
				Size:   Size,
				Offset: TotalOffset,
			})

			TotalOffset += Size
		}
	}

	for dirPath, dirMode := range dirMap {
		Dirs = append(Dirs, &Dir{
			Path: dirPath,
			Mode: uint32(dirMode),
		})
	}

	container := &Container{
		Size:     TotalOffset,
		Dirs:     Dirs,
		Symlinks: Symlinks,
		Files:    Files,
	}
	return container, nil
}

// Stats return a human-readable summary of the contents of a container
func (container *Container) Stats() string {
	return fmt.Sprintf("%d files, %d dirs, %d symlinks",
		len(container.Files), len(container.Dirs), len(container.Symlinks))
}

var _ fmt.Formatter = (*Container)(nil)

// Stats return a human-readable summary of the contents of a container
func (container *Container) Format(f fmt.State, c rune) {
	fmt.Fprintf(f, "%s (%s)",
		progress.FormatBytes(container.Size),
		container.Stats(),
	)
}

// IsSingleFile returns true if the container contains
// exactly one files, and no directories or symlinks.
func (container *Container) IsSingleFile() bool {
	if len(container.Files) == 1 && len(container.Dirs) == 0 && len(container.Symlinks) == 0 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "os"
import "crypto/tls"
import "crypto/x509"
import "encoding/pem"
import "flag"

func main() {

  var server = flag.String("server", "", "The TLS host (name or IP) to connect to")
  var port = flag.Int("port", 443, "The port the TLS service is running on")
  var network = flag.String("net", "tcp", "Connect to this kind of network: tcp, udp, or unix")
  var ipv4only = flag.Bool("4", false, "Only connect via IPv4")
  var ipv6only = flag.Bool("6", false, "Only connect via IPv6")
  flag.Parse()

  if (*server == "") {
    panic("Need to specify a server.")
  }

  if (*network != "tcp" && *network != "udp" && *network != "unix") {
    panic("Unknown kind of network type! Try tcp.")
  }

  var network_suffix string = ""
  if (*ipv4only && *ipv6only) {
    fmt.Println("Specifying both -4 & -6 is redundant")
  } else if (*ipv4only) {
    network_suffix = "4"
  } else if (*ipv6only) {
    network_suffix = "6"
  }

  target_host := fmt.Sprintf("%s:%d", *server, *port)
  network_type := fmt.Sprintf("%s%s", *network, network_suffix)

  conn, err := tls.Dial(network_type, target_host, &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()
  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    var pem_type string = ""

    switch cert.PublicKeyAlgorithm {
    case x509.RSA:
      pem_type = "RSA PUBLIC KEY"

    case x509.ECDSA:
      pem_type = "ECDSA PUBLIC KEY"

    case x509.DSA:
      pem_type = "DSA PUBLIC KEY"
    }

    pem_pubkey := &pem.Block{
      Type: pem_type,
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
}
<commit_msg>Refactor imports into a single call<commit_after>package main

import (
  "fmt"
  "os"
  "crypto/tls"
  "crypto/x509"
  "encoding/pem"
  "flag"
)

func main() {

  var server = flag.String("server", "", "The TLS host (name or IP) to connect to")
  var port = flag.Int("port", 443, "The port the TLS service is running on")
  var network = flag.String("net", "tcp", "Connect to this kind of network: tcp, udp, or unix")
  var ipv4only = flag.Bool("4", false, "Only connect via IPv4")
  var ipv6only = flag.Bool("6", false, "Only connect via IPv6")
  flag.Parse()

  if (*server == "") {
    panic("Need to specify a server.")
  }

  if (*network != "tcp" && *network != "udp" && *network != "unix") {
    panic("Unknown kind of network type! Try tcp.")
  }

  var network_suffix string = ""
  if (*ipv4only && *ipv6only) {
    fmt.Println("Specifying both -4 & -6 is redundant")
  } else if (*ipv4only) {
    network_suffix = "4"
  } else if (*ipv6only) {
    network_suffix = "6"
  }

  target_host := fmt.Sprintf("%s:%d", *server, *port)
  network_type := fmt.Sprintf("%s%s", *network, network_suffix)

  conn, err := tls.Dial(network_type, target_host, &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()
  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    var pem_type string = ""

    switch cert.PublicKeyAlgorithm {
    case x509.RSA:
      pem_type = "RSA PUBLIC KEY"

    case x509.ECDSA:
      pem_type = "ECDSA PUBLIC KEY"

    case x509.DSA:
      pem_type = "DSA PUBLIC KEY"
    }

    pem_pubkey := &pem.Block{
      Type: pem_type,
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
}
<|endoftext|>


<commit_before>package main

import "fmt"
//import "os"
//import "crypto/tls"
//import "crypto/x509"
//import "encoding/pem"
import "flag"

func main() {

  var server = flag.String("server", "mail.google.com", "The TLS host (name or IP) to connect to")
  var port = flag.Int("port", 443, "The port the TLS service is running on")
  flag.Parse()

  target_host := fmt.Sprintf("%s:%d", &server, &port)

  fmt.Println(target_host)

  /*
  conn, err := tls.Dial("tcp", target_host, &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()
  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    var pem_type string = ""

    switch cert.PublicKeyAlgorithm {
    case x509.RSA:
      pem_type = "RSA PUBLIC KEY"

    case x509.ECDSA:
      pem_type = "ECDSA PUBLIC KEY"

    case x509.DSA:
      pem_type = "DSA PUBLIC KEY"
    }

    pem_pubkey := &pem.Block{
      Type: pem_type,
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
  */
}
<commit_msg>a bunch of flag checking<commit_after>package main

import "fmt"
//import "os"
//import "crypto/tls"
//import "crypto/x509"
//import "encoding/pem"
import "flag"

func main() {

  var server = flag.String("server", "", "The TLS host (name or IP) to connect to")
  var port = flag.Int("port", 443, "The port the TLS service is running on")
  var network = flag.String("net", "tcp", "Connect to this kind of network: tcp, udp, or unix")
  var ipv4only = flag.Bool("4", false, "Only connect via IPv4")
  var ipv6only = flag.Bool("6", false, "Only connect via IPv6")
  flag.Parse()

  if (*server == "") {
    panic("Need to specify a server.")
  }

  if (*network != "tcp" && *network != "udp" && *network != "unix") {
    panic("Unknown kind of network type! Try tcp.")
  }

  if (*ipv4only && *ipv6only) {
    fmt.Println("Specifying both -4 & -6 is redundant, tls-grab will try both by default.")
  }

  var network_suffix string
  if (*ipv4only) {
    network_suffix = "4"
  } else if (*ipv6only) {
    network_suffix = "6"
  } else {
    network_suffix = ""
  }

  target_host := fmt.Sprintf("%s:%d via %s%s", *server, *port, *network, network_suffix)
  fmt.Println(target_host)

  /*
  conn, err := tls.Dial("tcp", target_host, &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()
  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    var pem_type string = ""

    switch cert.PublicKeyAlgorithm {
    case x509.RSA:
      pem_type = "RSA PUBLIC KEY"

    case x509.ECDSA:
      pem_type = "ECDSA PUBLIC KEY"

    case x509.DSA:
      pem_type = "DSA PUBLIC KEY"
    }

    pem_pubkey := &pem.Block{
      Type: pem_type,
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
  */
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/krasoffski/gomill/crawler"
)

// mozTop creates new reader which represents list of sites addresses from
// https://moz.com/top500 delimeted by new line charaster.
// Request timeout is 60s.
func mozTop(head int) io.Reader {
	// TODO: Create a better solution for providing top sites.

	client := http.Client{Timeout: time.Second * 60}
	fmt.Printf("Getting csv with top 500 from moz.com... ")
	start := time.Now()
	resp, err := client.Get("https://moz.com/top500/domains/csv")

	if err != nil {
		log.Fatalf("error performing request %s\n", err)
	}
	if resp.StatusCode != http.StatusOK {
		log.Fatalf("error getting moz top 500: %s\n", resp.Status)
	}
	if !strings.Contains(resp.Header.Get("Content-Type"), "text/csv") {
		log.Fatalln("error getting csv data, wrong conten type")
	}
	defer resp.Body.Close()

	var urls []string
	reader := csv.NewReader(resp.Body)

	for i := 0; i <= head; i++ {
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("error reading csv %s\n", err)
		}
		url := "http://" + strings.TrimRight(record[1], "/")

		urls = append(urls, url)
	}

	// Do not count strings join and new reader creation.
	fmt.Printf("done in %.2fs\n", time.Since(start).Seconds())
	// Skipping csv header.
	return strings.NewReader(strings.Join(urls[1:], "\n"))
}

func main() {
	workers := flag.Int("workers", 2, "number of workers (goroutines)")
	bufsize := flag.Int("bufsize", 0, "size of tasks' buffer")
	topsite := flag.Int("topsite", 0, "specify head of moz top sites, max 500")
	timeout := flag.Duration("timeout", 60*time.Second, "request timeout")
	threads := flag.Int("threads", 0, "numbers of OS threads, 0 runtime defaults")

	flag.Parse()

	var reader io.Reader = os.Stdin
	if *topsite > 0 {
		if *topsite < 500 {
			reader = mozTop(*topsite)
		} else {
			fmt.Fprintf(os.Stderr,
				"error value: %d, specify top sites within range [1, 500]\n",
				*topsite)
			os.Exit(1)
		}
	}
	// This call might be removed in the future.
	runtime.GOMAXPROCS(*threads)
	m := crawler.New(reader, *bufsize, &http.Client{Timeout: *timeout})
	start := time.Now()
	m.Run(*workers)
	fmt.Printf("%.2fs elapsed\n", time.Since(start).Seconds())
}
<commit_msg>fix rounding for crawler<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/krasoffski/gomill/crawler"
)

// mozTop creates new reader which represents list of sites addresses from
// https://moz.com/top500 delimeted by new line charaster.
// Request timeout is 60s.
func mozTop(head int) io.Reader {
	// TODO: Create a better solution for providing top sites.

	client := http.Client{Timeout: time.Second * 60}
	fmt.Printf("Getting csv with top 500 from moz.com... ")
	start := time.Now()
	resp, err := client.Get("https://moz.com/top500/domains/csv")

	if err != nil {
		log.Fatalf("error performing request %s\n", err)
	}
	if resp.StatusCode != http.StatusOK {
		log.Fatalf("error getting moz top 500: %s\n", resp.Status)
	}
	if !strings.Contains(resp.Header.Get("Content-Type"), "text/csv") {
		log.Fatalln("error getting csv data, wrong conten type")
	}
	defer resp.Body.Close()

	var urls []string
	reader := csv.NewReader(resp.Body)

	for i := 0; i <= head; i++ {
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("error reading csv %s\n", err)
		}
		url := "http://" + strings.TrimRight(record[1], "/")

		urls = append(urls, url)
	}

	// Do not count strings join and new reader creation.
	fmt.Printf("done in %.2fs\n", time.Since(start).Seconds())
	// Skipping csv header.
	return strings.NewReader(strings.Join(urls[1:], "\n"))
}

func main() {
	workers := flag.Int("workers", 2, "number of workers (goroutines)")
	bufsize := flag.Int("bufsize", 0, "size of tasks' buffer")
	topsite := flag.Int("topsite", 0, "specify head of moz top sites, max 500")
	timeout := flag.Duration("timeout", 60*time.Second, "request timeout")
	threads := flag.Int("threads", 0, "numbers of OS threads, 0 runtime defaults")

	flag.Parse()

	var reader io.Reader = os.Stdin
	if *topsite > 0 {
		if *topsite <= 500 {
			reader = mozTop(*topsite)
		} else {
			fmt.Fprintf(os.Stderr,
				"error value: %d, specify top sites within range [1, 500]\n",
				*topsite)
			os.Exit(1)
		}
	}
	// This call might be removed in the future.
	runtime.GOMAXPROCS(*threads)
	m := crawler.New(reader, *bufsize, &http.Client{Timeout: *timeout})
	start := time.Now()
	m.Run(*workers)
	done := time.Since(start)
	fmt.Printf("%v elapsed\n", done-(done%time.Millisecond))
}
<|endoftext|>


<commit_before>package api

import (
	//	"gopkg.in/mgo.v2"
	"appengine/aetest"
	"appengine/datastore"
	"github.com/ErikBjare/Futarchio/src/db"
	"log"
	"testing"
)

func TestUsers(t *testing.T) {
	c, err := aetest.NewContext(nil)
	log.Println("hello")
	if err != nil {
		t.Error(err)
	}
	defer c.Close()
	result := []db.User{}
	q := datastore.NewQuery("User").Filter("email =", "erik@bjareho.lt")
	var users []db.User
	_, err = q.GetAll(c, &users)
	if err != nil {
		t.Error(err)
	}
	if len(users) == 0 {
		t.Error("Couldn't find Erik in database")
	} else if len(result) > 1 {
		t.Error("More than one user with email erik@bjareho.lt in database")
	}
}

/*
func BenchmarkUserExistanceCycle(b *testing.B) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		panic(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		user := db.NewUser("tester", "password", "Tester", "test@example.com", []string{})
		userkey, err := datastore.Put(c, datastore.NewIncompleteKey(c, "User", nil), user)
		if err != nil {
			b.Error("Error when creating user")
			log.Println("I'm here")
		}
		datastore.Delete(c, userkey)
	}
}
*/

func TestNotDone(t *testing.T) {
	t.Skip("Not implemented")
}
<commit_msg>Shitty tests...<commit_after>package api

import (
	//	"gopkg.in/mgo.v2"
	"appengine"
	"appengine/aetest"
	"appengine/datastore"
	"encoding/json"
	"fmt"
	"github.com/ErikBjare/Futarchio/src/db"
	"io/ioutil"
	"log"
	"net/http"
	"testing"
)

func init() {
	http.Get("http://localhost:8080/api/0/init")
}

func TestAuth(t *testing.T) {
	inst, err := aetest.NewInstance(nil)
	if err != nil {
		t.Fatalf("Failed to create instance: %v", err)
	}
	defer inst.Close()

	c, err := aetest.NewContext(nil)
	hostname := appengine.DefaultVersionHostname(c)
	defer c.Close()

	log.Println(hostname)

	client := &http.Client{}
	req1, err := inst.NewRequest("GET", "http://"+hostname+"/api/0/init", nil)
	if err != nil {
		t.Fatalf("Failed to create req1: %v", err)
	}
	_, err = client.Do(req1)
	if err != nil {
		t.Fatal(err)
	}

	req2, err := inst.NewRequest("POST", "/api/0/auth", nil)
	req2.Header.Add("Content-Type", "application/json")
	req2.Body.Read([]byte("{\"username\": \"erb\", \"password\": \"password\"}"))
	if err != nil {
		t.Fatalf("Failed to create req1: %v", err)
	}
	resp, err := client.Do(req2)
	if err != nil {
		t.Fatal(err)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}

	msg := map[string]map[string]string{}
	log.Println(string(body))
	json.Unmarshal(body, &msg)

	urls := []string{"http://localhost:8080/api/0/users", "http://localhost:8080/api/0/users/me"}
	for _, url := range urls {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			t.Fatal(err)
		}
		req.Header.Add("Authorization", msg["auth"]["key"])
		resp, err = client.Do(req)
		if err != nil {
			t.Error("Is the server running?")
			t.Fatal(err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			t.Fatal(fmt.Sprintf("Status code was not 200, was %d with message: %s", resp.StatusCode, resp.Status))
		}

		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Error(err)
		}
		data := map[string]interface{}{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			t.Error(err)
		}
		if data["length"] == float64(0) {
			t.Fatal(fmt.Sprintf("Got zero results: %f", data["length"]))
		}
	}
}

func BenchmarkAPICall(b *testing.B) {
	client := &http.Client{}
	for i := 0; i < b.N; i++ {
		req, err := http.NewRequest("GET", "http://localhost:8080/api/0/users", nil)
		if err != nil {
			b.Fatal(err)
		}
		req.Header.Add("Authorization", "Basic YWRtaW46YWRtaW4=")
		resp, err := client.Do(req)
		if err != nil {
			b.Error("Is the server running?")
			b.Fatal(err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			b.Fatal(fmt.Sprintf("Status code was not 200, was %d with message: %s", resp.StatusCode, resp.Status))
		}

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			b.Error(err)
		}
		data := map[string]interface{}{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			b.Error(err)
		}
		if data["length"] == float64(0) {
			b.Fatal(fmt.Sprintf("Got zero results: %f", data["length"]))
		}
	}
}

func TestUsers(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Error(err)
	}
	defer c.Close()

	key := datastore.NewKey(c, "User", "", 1, nil)
	_, err = datastore.Put(c, key, db.NewUser("erb", "secretpassword", "Erik", "erik@bjareho.lt", []string{}))
	if err != nil {
		t.Fatal(err)
	}

	var user db.User
	err = datastore.Get(c, key, &user)
	if err != nil {
		t.Error(err)
	}

	var users []db.User
	q := datastore.NewQuery("User").Filter("Email =", "erik@bjareho.lt")
	keys, err := q.GetAll(c, &users)
	if err != nil {
		t.Error(err)
	}

	if len(keys) == 0 {
		t.Error("Couldn't find Erik in database")
	} else if len(keys) > 1 {
		t.Error("More than one user with email erik@bjareho.lt in database")
	}
}

/*
func BenchmarkUserExistanceCycle(b *testing.B) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		panic(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		user := db.NewUser("tester", "password", "Tester", "test@example.com", []string{})
		userkey, err := datastore.Put(c, datastore.NewIncompleteKey(c, "User", nil), user)
		if err != nil {
			b.Error("Error when creating user")
			log.Println("I'm here")
		}
		datastore.Delete(c, userkey)
	}
}
*/
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"flag"
	"strconv"
	"strings"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
	App       *App
	flagSet   *flag.FlagSet
	globalSet *flag.FlagSet
}

// Creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, globalSet *flag.FlagSet) *Context {
	return &Context{app, set, globalSet}
}

// Looks up the value of a local int flag, returns 0 if no int flag exists
func (c *Context) Int(name string) int {
	return lookupInt(name, c.flagSet)
}

// Looks up the value of a local float64 flag, returns 0 if no int flag exists
func (c *Context) Float64(name string) float64 {
	return lookupFloat64(name, c.flagSet)
}

// Looks up the value of a local bool flag, returns false if no bool flag exists
func (c *Context) Bool(name string) bool {
	return lookupBool(name, c.flagSet)
}

// Looks up the value of a local string flag, returns "" if no string flag exists
func (c *Context) String(name string) string {
	return lookupString(name, c.flagSet)
}

// Looks up the value of a local string slice flag, returns nil if no string slice flag exists
func (c *Context) StringSlice(name string) []string {
	return lookupStringSlice(name, c.flagSet)
}

// Looks up the value of a local int slice flag, returns nil if no int slice flag exists
func (c *Context) IntSlice(name string) []int {
	return lookupIntSlice(name, c.flagSet)
}

// Looks up the value of a global int flag, returns 0 if no int flag exists
func (c *Context) GlobalInt(name string) int {
	return lookupInt(name, c.globalSet)
}

// Looks up the value of a global bool flag, returns false if no bool flag exists
func (c *Context) GlobalBool(name string) bool {
	return lookupBool(name, c.globalSet)
}

// Looks up the value of a global string flag, returns "" if no string flag exists
func (c *Context) GlobalString(name string) string {
	return lookupString(name, c.globalSet)
}

// Looks up the value of a global string slice flag, returns nil if no string slice flag exists
func (c *Context) GlobalStringSlice(name string) []string {
	return lookupStringSlice(name, c.globalSet)
}

// Looks up the value of a global int slice flag, returns nil if no int slice flag exists
func (c *Context) GlobalIntSlice(name string) []int {
	return lookupIntSlice(name, c.globalSet)
}

type Args []string

// Returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	args := Args(c.flagSet.Args())
	return args
}

// Returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
	if len(a) > n {
		return a[n]
	}
	return ""
}

// Returns the first argument, or else a blank string
func (a Args) First() string {
	return a.Get(0)
}

// Return the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
	if len(a) >= 2 {
		return []string(a)[1:]
	}
	return []string{}
}

// Checks if there are any arguments present
func (a Args) Present() bool {
	return len(a) != 0
}

func lookupInt(name string, set *flag.FlagSet) int {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.Atoi(f.Value.String())
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupFloat64(name string, set *flag.FlagSet) float64 {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseFloat(f.Value.String(), 64)
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupString(name string, set *flag.FlagSet) string {
	f := set.Lookup(name)
	if f != nil {
		return f.Value.String()
	}

	return ""
}

func lookupStringSlice(name string, set *flag.FlagSet) []string {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*StringSlice)).Value()

	}

	return nil
}

func lookupIntSlice(name string, set *flag.FlagSet) []int {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*IntSlice)).Value()

	}

	return nil
}

func lookupBool(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return false
		}
		return val
	}

	return false
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := strings.Split(f.getName(), ",")
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			set.Set(name, ff.Value.String())
		}
	}
	return nil
}
<commit_msg>Fix typo in godoc for Float64<commit_after>package cli

import (
	"errors"
	"flag"
	"strconv"
	"strings"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
	App       *App
	flagSet   *flag.FlagSet
	globalSet *flag.FlagSet
}

// Creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, globalSet *flag.FlagSet) *Context {
	return &Context{app, set, globalSet}
}

// Looks up the value of a local int flag, returns 0 if no int flag exists
func (c *Context) Int(name string) int {
	return lookupInt(name, c.flagSet)
}

// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists
func (c *Context) Float64(name string) float64 {
	return lookupFloat64(name, c.flagSet)
}

// Looks up the value of a local bool flag, returns false if no bool flag exists
func (c *Context) Bool(name string) bool {
	return lookupBool(name, c.flagSet)
}

// Looks up the value of a local string flag, returns "" if no string flag exists
func (c *Context) String(name string) string {
	return lookupString(name, c.flagSet)
}

// Looks up the value of a local string slice flag, returns nil if no string slice flag exists
func (c *Context) StringSlice(name string) []string {
	return lookupStringSlice(name, c.flagSet)
}

// Looks up the value of a local int slice flag, returns nil if no int slice flag exists
func (c *Context) IntSlice(name string) []int {
	return lookupIntSlice(name, c.flagSet)
}

// Looks up the value of a global int flag, returns 0 if no int flag exists
func (c *Context) GlobalInt(name string) int {
	return lookupInt(name, c.globalSet)
}

// Looks up the value of a global bool flag, returns false if no bool flag exists
func (c *Context) GlobalBool(name string) bool {
	return lookupBool(name, c.globalSet)
}

// Looks up the value of a global string flag, returns "" if no string flag exists
func (c *Context) GlobalString(name string) string {
	return lookupString(name, c.globalSet)
}

// Looks up the value of a global string slice flag, returns nil if no string slice flag exists
func (c *Context) GlobalStringSlice(name string) []string {
	return lookupStringSlice(name, c.globalSet)
}

// Looks up the value of a global int slice flag, returns nil if no int slice flag exists
func (c *Context) GlobalIntSlice(name string) []int {
	return lookupIntSlice(name, c.globalSet)
}

type Args []string

// Returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	args := Args(c.flagSet.Args())
	return args
}

// Returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
	if len(a) > n {
		return a[n]
	}
	return ""
}

// Returns the first argument, or else a blank string
func (a Args) First() string {
	return a.Get(0)
}

// Return the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
	if len(a) >= 2 {
		return []string(a)[1:]
	}
	return []string{}
}

// Checks if there are any arguments present
func (a Args) Present() bool {
	return len(a) != 0
}

func lookupInt(name string, set *flag.FlagSet) int {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.Atoi(f.Value.String())
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupFloat64(name string, set *flag.FlagSet) float64 {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseFloat(f.Value.String(), 64)
		if err != nil {
			return 0
		}
		return val
	}

	return 0
}

func lookupString(name string, set *flag.FlagSet) string {
	f := set.Lookup(name)
	if f != nil {
		return f.Value.String()
	}

	return ""
}

func lookupStringSlice(name string, set *flag.FlagSet) []string {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*StringSlice)).Value()

	}

	return nil
}

func lookupIntSlice(name string, set *flag.FlagSet) []int {
	f := set.Lookup(name)
	if f != nil {
		return (f.Value.(*IntSlice)).Value()

	}

	return nil
}

func lookupBool(name string, set *flag.FlagSet) bool {
	f := set.Lookup(name)
	if f != nil {
		val, err := strconv.ParseBool(f.Value.String())
		if err != nil {
			return false
		}
		return val
	}

	return false
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := strings.Split(f.getName(), ",")
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			set.Set(name, ff.Value.String())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "fmt"
  "strconv"
  "image"
  "image/png"
  "image/color"
  "math"
  "math/rand"
)

// Our functions
func f0(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return x / 2, y / 2
}

func f1(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return (x + 1) / 2, y / 2
}

func f2(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return x / 2, (y + 1) / 2
}

func f3(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Sin(x), math.Sin(y)
}

func f4(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Cos(x), math.Cos(y)
}

func f5(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Cos(a*x + b*y + c), math.Cos(d*x + e*y + f)
}

func f6(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Sin(a*x + b*y + c), math.Sin(d*x + e*y + f)
}

func f7(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Sin(a*x + b*y + c), math.Cos(d*x + e*y + f)
}

func f8(x, y, a, b, c, d, e, f float64) (float64, float64) {
  one := a*x+b*y+c
  two := d*x + e*y + f
  return math.Sin(one) * math.Sin(one), math.Sin(two) * math.Sin(two)
}

func f9(x, y, a, b, c, d, e, f float64) (float64, float64) {
  one := a*x+b*y+c
  two := d*x + e*y + f
  return math.Cos(one) * math.Cos(one), math.Cos(two) * math.Cos(two)
}

func f10(x, y, a, b, c, d, e, f float64) (float64, float64) {
  one := a*x+b*y+c
  two := d*x + e*y + f
  return math.Sin(one) * math.Cos(one), math.Sin(two) * math.Cos(two)
}

// get the third-largest value in a matrix. I can probably do this better
func equalize(values int, arr [][]int) [][]int {
  mx := 0
  total := len(arr) * len(arr[0])
  for _, row := range arr {
    for _, v := range row {
      if mx < v {
	mx = v
      }
    }
  }
  hist := make([]int, mx + 1)
  for _, row := range arr {
    for _, v := range row {
      hist[v] += 1
    }
  }
  min := 0
  max := 0
  top := int(float64(total) * .9995)
  bottom := int(float64(total) * .0005)
  count := 0
  // fmt.Printf("Eq", mx, total, top, bottom)
  // fmt.Printf("Hist", hist)
  for i, n := range hist {
    count += n
    if count > bottom && min == 0 {
      min = i
    }
    if count > top {
      max = i
      break
    }
  }
  // fmt.Printf("Eq", mx, total, min, max)
  for i := range arr {
    for j := range arr[i] {
      if max == min {
	arr[i][j] = 0
	continue
      }
      if arr[i][j] > max {
	arr[i][j] = max
      }
      if arr[i][j] < min {
	arr[i][j] = min
      }
      arr[i][j] = values * (arr[i][j] - min) / (max - min)
    }
  }
  return arr
}

/*
  (x, y) = a random point in the biunit square
  iterate {
  i = a random integer from 0 to n  1 inclusive
  (x, y) = Fi(x, y)
  plot (x, y) except during the first 20 iterations
  }
*/

func flame(width, height, iters int, usefuncs []int) *image.RGBA {
  x := rand.Float64()*2 - 1
  y := rand.Float64()*2 - 1
  mx := make([][]int, height)
  for y := range mx {
    mx[y] = make([]int, width)
    for x := range mx[y] {
      mx[y][x] = 0
    }
  }
  var a, b, c, d, e, f float64
  // these are our parameters
  a, b, c, d, e, f = 1, 2, 1, 1, 4, 5
  allfuncs := [11]func(float64, float64, float64, float64, float64, float64, float64, float64) (float64, float64){f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10}
  // and the F_i s that we'll be using
  funcs := make([]func(float64, float64, float64, float64, float64, float64, float64, float64) (float64, float64), len(usefuncs))
  for i, v := range(usefuncs) {
    funcs[i] = allfuncs[v]
  }
  for at := 0; at < iters; at++ {
    x, y = funcs[rand.Intn(len(funcs))](x, y, a, b, c, d, e, f)
    // I should probably refactor this
    if x < -1 {
      x = -1
      continue
    }
    if x > 1 {
      x = 1
      continue
    }
    if y < -1 {
      y = -1
      continue
    }
    if y > 1 {
      y = 1
      continue
    }
    //fmt.Println("after", x,y)
    if at < 20 {
      continue
    }
    // refactor, make more readable
    mx[int((y+1)/2*float64(height-1))][int((x+1)/2*float64(width-1))] += 1
  }
  mx = equalize(255, mx)
  m := image.NewRGBA(image.Rect(0, 0, width, height))
  // now write the values to an image, equalized by the 3rd-brightest point
  for x, row := range mx {
    for y, v := range row {
      val := uint8(v)
      m.Set(x, y, color.RGBA{val, val * 100 / 255, val, 255})
    }
  }
  return m
}

func writeit(w, h, i int, use []int) {
  m := flame(w, h, i, use)
  name := "smallish-"
  num := ""
  for _, z := range use {
    num = strconv.Itoa(z) + num
  }
  toimg, _ := os.Create(name + num + ".png")
  defer toimg.Close()

  png.Encode(toimg, m)
}

func main() {
  w := 400
  h := 400
  i := 1000000
  all := int(math.Pow(2, 11))
  for z := all/4; z < all; z++ {
    n := 0
    for s := z; s > 0; s >>= 1 {
      if s % 2 == 1 {
	n += 1
      }
    }
    use := make([]int, n)
    n = 0
    a := 0
    for s := z; s > 0; s >>= 1 {
      if s % 2 == 1 {
	use[n] = a
	n += 1
      }
      a += 1
    }
    fmt.Println("Yeah", use)
    writeit(w, h, i, use)
  }
}
<commit_msg>refactor; separate generation from render<commit_after>package main

import (
  "os"
  "fmt"
  "strconv"
  "image"
  "image/png"
  "image/color"
  "math"
  "math/rand"
)

// Our functions
func f0(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return x / 2, y / 2
}

func f1(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return (x + 1) / 2, y / 2
}

func f2(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return x / 2, (y + 1) / 2
}

func f3(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Sin(x), math.Sin(y)
}

func f4(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Cos(x), math.Cos(y)
}

func f5(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Cos(a*x + b*y + c), math.Cos(d*x + e*y + f)
}

func f6(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Sin(a*x + b*y + c), math.Sin(d*x + e*y + f)
}

func f7(x, y, a, b, c, d, e, f float64) (float64, float64) {
  return math.Sin(a*x + b*y + c), math.Cos(d*x + e*y + f)
}

func f8(x, y, a, b, c, d, e, f float64) (float64, float64) {
  one := a*x+b*y+c
  two := d*x + e*y + f
  return math.Sin(one) * math.Sin(one), math.Sin(two) * math.Sin(two)
}

func f9(x, y, a, b, c, d, e, f float64) (float64, float64) {
  one := a*x+b*y+c
  two := d*x + e*y + f
  return math.Cos(one) * math.Cos(one), math.Cos(two) * math.Cos(two)
}

func f10(x, y, a, b, c, d, e, f float64) (float64, float64) {
  one := a*x+b*y+c
  two := d*x + e*y + f
  return math.Sin(one) * math.Cos(one), math.Sin(two) * math.Cos(two)
}

// get the third-largest value in a matrix. I can probably do this better
func equalize(values int, arr [][]int, maxp, minp float64) [][]int {
  mx := 0
  total := len(arr) * len(arr[0])
  for _, row := range arr {
    for _, v := range row {
      if mx < v {
	mx = v
      }
    }
  }
  hist := make([]int, mx + 1)
  for _, row := range arr {
    for _, v := range row {
      hist[v] += 1
    }
  }
  min := 0
  max := 0
  top := int(float64(total) * maxp)
  bottom := int(float64(total) * minp)
  count := 0
  // fmt.Printf("Eq", mx, total, top, bottom)
  // fmt.Printf("Hist", hist)
  for i, n := range hist {
    count += n
    if count > bottom && min == 0 {
      min = i
    }
    if count > top {
      max = i
      break
    }
  }
  // fmt.Printf("Eq", mx, total, min, max)
  for i := range arr {
    for j := range arr[i] {
      if max == min {
	arr[i][j] = 0
	continue
      }
      if arr[i][j] > max {
	arr[i][j] = max
      }
      if arr[i][j] < min {
	arr[i][j] = min
      }
      arr[i][j] = values * (arr[i][j] - min) / (max - min)
    }
  }
  return arr
}

/*
  (x, y) = a random point in the biunit square
  iterate {
  i = a random integer from 0 to n  1 inclusive
  (x, y) = Fi(x, y)
  plot (x, y) except during the first 20 iterations
  }
*/

type Point struct {
  x, y float64
}

func flame(width, height, iters int, usefuncs []int) *image.RGBA {
  x := rand.Float64()*2 - 1
  y := rand.Float64()*2 - 1
  data := make([]Point, iters)
    // refactor, make more readable
  var a, b, c, d, e, f float64
  // these are our parameters
  a, b, c, d, e, f = 1, 2, 1, 1, 4, 5
  allfuncs := [11]func(float64, float64,
                       float64, float64, float64,
                       float64, float64, float64) (float64, float64) {
                         f0, f1, f2, f3, f4,
                         f5, f6, f7, f8, f9, f10}
  // and the F_i s that we'll be using
  funcs := make([]func(float64, float64,
                       float64, float64, float64,
                       float64, float64, float64) (float64, float64),
                len(usefuncs))
  for i, v := range(usefuncs) {
    funcs[i] = allfuncs[v]
  }
  for at := 0; at < iters; at++ {
    x, y = funcs[rand.Intn(len(funcs))](x, y, a, b, c, d, e, f)
    /*
    // I should probably refactor this
    if x < -1 || x > 1 || y < -1 || y > 1 {
      continue
    }
    */
    //fmt.Println("after", x,y)
    if at < 20 {
      continue
    }
    data[at] = Point{x, y}
    // mx[int((y+1)/2*float64(height-1))][int((x+1)/2*float64(width-1))] += 1
  }
  fmt.Println("Render")
  // now render
  mx := make([][]int, height)
  for y := range mx {
    mx[y] = make([]int, width)
    for x := range mx[y] {
      mx[y][x] = 0
    }
  }
  for _, v := range data {
    mx[int((v.y+1)/2*float64(height-1))][int((v.x+1)/2*float64(width-1))] += 1
  }
  mx = equalize(255, mx, .995, .0005)
  m := image.NewRGBA(image.Rect(0, 0, width, height))
  // now write the values to an image, equalized by the 3rd-brightest point
  for x, row := range mx {
    for y, v := range row {
      val := uint8(v)
      m.Set(x, y, color.RGBA{val, val * 100 / 255, val, 255})
    }
  }
  return m
}

func writeit(w, h, i int, use []int) {
  m := flame(w, h, i, use)
  name := "smallish-"
  num := ""
  for _, z := range use {
    num = strconv.Itoa(z) + num
  }
  toimg, _ := os.Create(name + num + ".png")
  defer toimg.Close()

  png.Encode(toimg, m)
}

func main() {
  w := 400
  h := 400
  i := 1000000
  //all := int(math.Pow(2, 11))
  fmt.Println("Generating")
  writeit(w, h, i, []int{7,6,5})
  /*
  for z := all/4; z < all; z++ {
    n := 0
    for s := z; s > 0; s >>= 1 {
      if s % 2 == 1 {
	n += 1
      }
    }
    use := make([]int, n)
    n = 0
    a := 0
    for s := z; s > 0; s >>= 1 {
      if s % 2 == 1 {
	use[n] = a
	n += 1
      }
      a += 1
    }
    fmt.Println("Yeah", use)
    writeit(w, h, i, use)
  }
  */
}
<|endoftext|>


<commit_before>package token

type TokenID int

const (
	ENDMARKER TokenID = iota
	NAME
	NUMBER
	STRING
	NEWLINE
	INDENT
	DEDENT
	LPAR
	RPAR
	LSQB
	RSQB
	COLON
	COMMA
	SEMI
	PLUS
	MINUS
	STAR
	SLASH
	VBAR
	AMPER
	LESS
	GREATER
	EQUAL
	DOT
	PERCENT
	LBRACE
	RBRACE
	EQEQUAL
	NOTEQUAL
	LESSEQUAL
	GREATEREQUAL
	TILDE
	CIRCUMFLEX
	LEFTSHIFT
	RIGHTSHIFT
	DOUBLESTAR
	PLUSEQUAL
	MINEQUAL
	STAREQUAL
	SLASHEQUAL
	PERCENTEQUAL
	AMPEREQUAL
	VBAREQUAL
	CIRCUMFLEXEQUAL
	LEFTSHIFTEQUAL
	RIGHTSHIFTEQUAL
	DOUBLESTAREQUAL
	DOUBLESLASH
	DOUBLESLASHEQUAL
	AT
	ATEQUAL
	RARROW
	ELLIPSIS
	OP
	AWAIT
	ASYNC
	ERRORTOKEN
	N_TOKENS
)
<commit_msg>add String() method for TokenID<commit_after>package token

type TokenID int

const (
	ENDMARKER TokenID = iota
	NAME
	NUMBER
	STRING
	NEWLINE
	INDENT
	DEDENT
	LPAR
	RPAR
	LSQB
	RSQB
	COLON
	COMMA
	SEMI
	PLUS
	MINUS
	STAR
	SLASH
	VBAR
	AMPER
	LESS
	GREATER
	EQUAL
	DOT
	PERCENT
	LBRACE
	RBRACE
	EQEQUAL
	NOTEQUAL
	LESSEQUAL
	GREATEREQUAL
	TILDE
	CIRCUMFLEX
	LEFTSHIFT
	RIGHTSHIFT
	DOUBLESTAR
	PLUSEQUAL
	MINEQUAL
	STAREQUAL
	SLASHEQUAL
	PERCENTEQUAL
	AMPEREQUAL
	VBAREQUAL
	CIRCUMFLEXEQUAL
	LEFTSHIFTEQUAL
	RIGHTSHIFTEQUAL
	DOUBLESTAREQUAL
	DOUBLESLASH
	DOUBLESLASHEQUAL
	AT
	ATEQUAL
	RARROW
	ELLIPSIS
	OP
	AWAIT
	ASYNC
	ERRORTOKEN
	N_TOKENS
)

func (id TokenID) String() string {
	return TokenNames[id]
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployer

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

const (
	defaultJobName = "pull-kops-e2e-kubernetes-aws"
	defaultGCSPath = "gcs://kops-ci/pulls/%v"
)

func (d *deployer) Build() error {
	if err := d.init(); err != nil {
		return err
	}
	if err := d.BuildOptions.Build(); err != nil {
		return err
	}
	return nil
}

func (d *deployer) verifyBuildFlags() error {
	if d.KopsRoot == "" {
		return errors.New("required kops-root when building from source")
	}
	if d.StageLocation != "" {
		if !strings.HasPrefix(d.StageLocation, "gs://") {
			return errors.New("stage-location must be a gs:// path")
		}
	} else {
		jobName := os.Getenv("JOB_NAME")
		if jobName == "" {
			jobName = defaultJobName
		}
		d.StageLocation = fmt.Sprintf(defaultGCSPath, jobName)
	}
	fi, err := os.Stat(d.KopsRoot)
	if err != nil {
		return err
	}
	if !fi.Mode().IsDir() {
		return errors.New("kops-root must be a directory")
	}

	d.BuildOptions.KopsRoot = d.KopsRoot
	d.BuildOptions.StageLocation = d.StageLocation
	return nil
}
<commit_msg>Set default --kops-root value when GOPATH is set<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployer

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strings"
)

const (
	defaultJobName = "pull-kops-e2e-kubernetes-aws"
	defaultGCSPath = "gs://kops-ci/pulls/%v"
)

func (d *deployer) Build() error {
	if err := d.init(); err != nil {
		return err
	}
	if err := d.BuildOptions.Build(); err != nil {
		return err
	}
	return nil
}

func (d *deployer) verifyBuildFlags() error {
	if d.KopsRoot == "" {
		if goPath := os.Getenv("GOPATH"); goPath != "" {
			d.KopsRoot = path.Join(goPath, "src", "k8s.io", "kops")
		} else {
			return errors.New("required kops-root when building from source")
		}
	}
	if d.StageLocation != "" {
		if !strings.HasPrefix(d.StageLocation, "gs://") {
			return errors.New("stage-location must be a gs:// path")
		}
	} else {
		jobName := os.Getenv("JOB_NAME")
		if jobName == "" {
			jobName = defaultJobName
		}
		d.StageLocation = fmt.Sprintf(defaultGCSPath, jobName)
	}
	fi, err := os.Stat(d.KopsRoot)
	if err != nil {
		return err
	}
	if !fi.Mode().IsDir() {
		return errors.New("kops-root must be a directory")
	}

	d.BuildOptions.KopsRoot = d.KopsRoot
	d.BuildOptions.StageLocation = d.StageLocation
	return nil
}
<|endoftext|>


<commit_before>package chunk

import (
	"fmt"
	"math/rand"
	"net/url"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"

	"github.com/weaveworks/common/instrument"
	"github.com/weaveworks/cortex/util"
)

const (
	hashKey  = "h"
	rangeKey = "r"
	chunkKey = "c"

	// For dynamodb errors
	tableNameLabel   = "table"
	errorReasonLabel = "error"
	otherError       = "other"

	provisionedThroughputExceededException = "ProvisionedThroughputExceededException"

	// Backoff for dynamoDB requests, to match AWS lib - see:
	// https://github.com/aws/aws-sdk-go/blob/master/service/dynamodb/customizations.go
	minBackoff = 50 * time.Millisecond
	maxBackoff = 50 * time.Second
	maxRetries = 20

	// See http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html.
	dynamoMaxBatchSize = 25
)

var (
	dynamoRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "dynamo_request_duration_seconds",
		Help:      "Time spent doing DynamoDB requests.",

		// DynamoDB latency seems to range from a few ms to a few sec and is
		// important.  So use 8 buckets from 64us to 8s.
		Buckets: prometheus.ExponentialBuckets(0.000128, 4, 8),
	}, []string{"operation", "status_code"})
	dynamoConsumedCapacity = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "dynamo_consumed_capacity_total",
		Help:      "The capacity units consumed by operation.",
	}, []string{"operation"})
	dynamoFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "dynamo_failures_total",
		Help:      "The total number of errors while storing chunks to the chunk store.",
	}, []string{tableNameLabel, errorReasonLabel})
	dynamoUnprocessedItems = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "dynamo_unprocessed_items_total",
		Help:      "Unprocessed items",
	})
)

func init() {
	prometheus.MustRegister(dynamoRequestDuration)
	prometheus.MustRegister(dynamoConsumedCapacity)
	prometheus.MustRegister(dynamoFailures)
	prometheus.MustRegister(dynamoUnprocessedItems)
}

type dynamoClientAdapter struct {
	DynamoDB dynamodbiface.DynamoDBAPI
}

// NewDynamoDBClient makes a new DynamoDBClient
func NewDynamoDBClient(dynamoDBURL string) (StorageClient, string, error) {
	url, err := url.Parse(dynamoDBURL)
	if err != nil {
		return nil, "", err
	}

	dynamoDBConfig, err := awsConfigFromURL(url)
	if err != nil {
		return nil, "", err
	}

	dynamoDBClient := dynamoClientAdapter{dynamodb.New(session.New(dynamoDBConfig))}
	tableName := strings.TrimPrefix(url.Path, "/")
	return dynamoDBClient, tableName, nil
}

func (d dynamoClientAdapter) NewWriteBatch() WriteBatch {
	return dynamoDBWriteBatch(map[string][]*dynamodb.WriteRequest{})
}

// batchWrite writes requests to the underlying storage, handling retires and backoff.
func (d dynamoClientAdapter) BatchWrite(ctx context.Context, input WriteBatch) error {
	outstanding := input.(dynamoDBWriteBatch)
	unprocessed := map[string][]*dynamodb.WriteRequest{}
	backoff, numRetries := minBackoff, 0
	for dictLen(outstanding)+dictLen(unprocessed) > 0 && numRetries < maxRetries {
		reqs := map[string][]*dynamodb.WriteRequest{}
		takeReqs(unprocessed, reqs, dynamoMaxBatchSize)
		takeReqs(outstanding, reqs, dynamoMaxBatchSize)
		var resp *dynamodb.BatchWriteItemOutput

		err := instrument.TimeRequestHistogram(ctx, "DynamoDB.BatchWriteItem", dynamoRequestDuration, func(_ context.Context) error {
			var err error
			resp, err = d.DynamoDB.BatchWriteItem(&dynamodb.BatchWriteItemInput{
				RequestItems:           reqs,
				ReturnConsumedCapacity: aws.String(dynamodb.ReturnConsumedCapacityTotal),
			})
			return err
		})
		for _, cc := range resp.ConsumedCapacity {
			dynamoConsumedCapacity.WithLabelValues("DynamoDB.BatchWriteItem").
				Add(float64(*cc.CapacityUnits))
		}

		if err != nil {
			for tableName := range reqs {
				recordDynamoError(tableName, err)
			}
		}

		// If there are unprocessed items, backoff and retry those items.
		if unprocessedItems := resp.UnprocessedItems; unprocessedItems != nil && dictLen(unprocessedItems) > 0 {
			takeReqs(unprocessedItems, unprocessed, -1)
			time.Sleep(backoff)
			backoff = nextBackoff(backoff)
			continue
		}

		// If we get provisionedThroughputExceededException, then no items were processed,
		// so back off and retry all.
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == provisionedThroughputExceededException {
			takeReqs(reqs, unprocessed, -1)
			time.Sleep(backoff)
			backoff = nextBackoff(backoff)
			numRetries++
			continue
		}

		// All other errors are fatal.
		if err != nil {
			return err
		}

		backoff = minBackoff
		numRetries = 0
	}

	if valuesLeft := dictLen(outstanding) + dictLen(unprocessed); valuesLeft > 0 {
		return fmt.Errorf("failed to write chunk after %d retries, %d values remaining", numRetries, valuesLeft)
	}
	return nil
}

func (d dynamoClientAdapter) QueryPages(ctx context.Context, entry IndexEntry, callback func(result ReadBatch, lastPage bool) (shouldContinue bool)) error {
	input := &dynamodb.QueryInput{
		TableName: aws.String(entry.TableName),
		KeyConditions: map[string]*dynamodb.Condition{
			hashKey: {
				AttributeValueList: []*dynamodb.AttributeValue{
					{S: aws.String(entry.HashValue)},
				},
				ComparisonOperator: aws.String(dynamodb.ComparisonOperatorEq),
			},
		},
		ReturnConsumedCapacity: aws.String(dynamodb.ReturnConsumedCapacityTotal),
	}

	if entry.RangeValuePrefix != nil {
		input.KeyConditions[rangeKey] = &dynamodb.Condition{
			AttributeValueList: []*dynamodb.AttributeValue{
				{B: entry.RangeValuePrefix},
			},
			ComparisonOperator: aws.String(dynamodb.ComparisonOperatorBeginsWith),
		}
	} else if entry.RangeValueStart != nil {
		input.KeyConditions[rangeKey] = &dynamodb.Condition{
			AttributeValueList: []*dynamodb.AttributeValue{
				{B: entry.RangeValueStart},
			},
			ComparisonOperator: aws.String(dynamodb.ComparisonOperatorGe),
		}
	}

	request, _ := d.DynamoDB.QueryRequest(input)
	backoff := minBackoff
	for page := request; page != nil; page = page.NextPage() {
		err := instrument.TimeRequestHistogram(ctx, "DynamoDB.QueryPages", dynamoRequestDuration, func(_ context.Context) error {
			return page.Send()
		})

		if cc := page.Data.(*dynamodb.QueryOutput).ConsumedCapacity; cc != nil {
			dynamoConsumedCapacity.WithLabelValues("DynamoDB.QueryPages").
				Add(float64(*cc.CapacityUnits))
		}

		if err != nil {
			recordDynamoError(*input.TableName, err)

			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == provisionedThroughputExceededException {
				time.Sleep(backoff)
				backoff = nextBackoff(backoff)
				continue
			}

			return page.Error
		}

		queryOutput := page.Data.(*dynamodb.QueryOutput)
		if getNextPage := callback(dynamoDBReadBatch(queryOutput.Items), !page.HasNextPage()); !getNextPage {
			return page.Error
		}

		backoff = minBackoff
	}

	return nil
}

func (d dynamoClientAdapter) ListTables() ([]string, error) {
	table := []string{}
	if err := d.DynamoDB.ListTablesPages(&dynamodb.ListTablesInput{}, func(resp *dynamodb.ListTablesOutput, _ bool) bool {
		for _, s := range resp.TableNames {
			table = append(table, *s)
		}
		return true
	}); err != nil {
		return nil, err
	}
	return table, nil
}

func (d dynamoClientAdapter) CreateTable(name string, readCapacity, writeCapacity int64) error {
	input := &dynamodb.CreateTableInput{
		TableName: aws.String(name),
		AttributeDefinitions: []*dynamodb.AttributeDefinition{
			{
				AttributeName: aws.String(hashKey),
				AttributeType: aws.String(dynamodb.ScalarAttributeTypeS),
			},
			{
				AttributeName: aws.String(rangeKey),
				AttributeType: aws.String(dynamodb.ScalarAttributeTypeB),
			},
		},
		KeySchema: []*dynamodb.KeySchemaElement{
			{
				AttributeName: aws.String(hashKey),
				KeyType:       aws.String(dynamodb.KeyTypeHash),
			},
			{
				AttributeName: aws.String(rangeKey),
				KeyType:       aws.String(dynamodb.KeyTypeRange),
			},
		},
		ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
			ReadCapacityUnits:  aws.Int64(readCapacity),
			WriteCapacityUnits: aws.Int64(writeCapacity),
		},
	}
	_, err := d.DynamoDB.CreateTable(input)
	return err
}

func (d dynamoClientAdapter) DescribeTable(name string) (readCapacity, writeCapacity int64, status string, err error) {
	out, err := d.DynamoDB.DescribeTable(&dynamodb.DescribeTableInput{
		TableName: aws.String(name),
	})
	if err != nil {
		return 0, 0, "", err
	}

	return *out.Table.ProvisionedThroughput.ReadCapacityUnits, *out.Table.ProvisionedThroughput.WriteCapacityUnits, *out.Table.TableStatus, nil
}

func (d dynamoClientAdapter) UpdateTable(name string, readCapacity, writeCapacity int64) error {
	_, err := d.DynamoDB.UpdateTable(&dynamodb.UpdateTableInput{
		TableName: aws.String(name),
		ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
			ReadCapacityUnits:  aws.Int64(readCapacity),
			WriteCapacityUnits: aws.Int64(writeCapacity),
		},
	})
	return err
}

type dynamoDBWriteBatch map[string][]*dynamodb.WriteRequest

func (b dynamoDBWriteBatch) Add(tableName, hashValue string, rangeValue []byte) {
	b[tableName] = append(b[tableName], &dynamodb.WriteRequest{
		PutRequest: &dynamodb.PutRequest{
			Item: map[string]*dynamodb.AttributeValue{
				hashKey:  {S: aws.String(hashValue)},
				rangeKey: {B: rangeValue},
			},
		},
	})
}

type dynamoDBReadBatch []map[string]*dynamodb.AttributeValue

func (b dynamoDBReadBatch) Len() int {
	return len(b)
}

func (b dynamoDBReadBatch) RangeValue(i int) []byte {
	return b[i][rangeKey].B
}

func (b dynamoDBReadBatch) Value(i int) []byte {
	chunkValue, ok := b[i][chunkKey]
	if !ok {
		return nil
	}
	return chunkValue.B
}

func nextBackoff(lastBackoff time.Duration) time.Duration {
	// Based on the "Decorrelated Jitter" approach from https://www.awsarchitectureblog.com/2015/03/backoff.html
	// sleep = min(cap, random_between(base, sleep * 3))
	backoff := minBackoff + time.Duration(rand.Int63n(int64((lastBackoff*3)-minBackoff)))
	if backoff > maxBackoff {
		backoff = maxBackoff
	}
	return backoff
}

func recordDynamoError(tableName string, err error) {
	if awsErr, ok := err.(awserr.Error); ok {
		dynamoFailures.WithLabelValues(tableName, awsErr.Code()).Add(float64(1))
	} else {
		dynamoFailures.WithLabelValues(tableName, otherError).Add(float64(1))
	}
}

func dictLen(b map[string][]*dynamodb.WriteRequest) int {
	result := 0
	for _, reqs := range b {
		result += len(reqs)
	}
	return result
}

// Fill 'to' with WriteRequests from 'from' until 'to' has at most max requests. Remove those requests from 'from'.
func takeReqs(from, to map[string][]*dynamodb.WriteRequest, max int) {
	outLen, inLen := dictLen(to), dictLen(from)
	toFill := inLen
	if max > 0 {
		toFill = util.Min(inLen, max-outLen)
	}
	for toFill > 0 {
		for tableName, fromReqs := range from {
			taken := util.Min(len(fromReqs), toFill)
			if taken > 0 {
				to[tableName] = append(to[tableName], fromReqs[:taken]...)
				from[tableName] = fromReqs[taken:]
				toFill -= taken
			}
		}
	}
}
<commit_msg>Removed not used metric 'dynamo_unprocessed_items_total' (#340)<commit_after>package chunk

import (
	"fmt"
	"math/rand"
	"net/url"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"

	"github.com/weaveworks/common/instrument"
	"github.com/weaveworks/cortex/util"
)

const (
	hashKey  = "h"
	rangeKey = "r"
	chunkKey = "c"

	// For dynamodb errors
	tableNameLabel   = "table"
	errorReasonLabel = "error"
	otherError       = "other"

	provisionedThroughputExceededException = "ProvisionedThroughputExceededException"

	// Backoff for dynamoDB requests, to match AWS lib - see:
	// https://github.com/aws/aws-sdk-go/blob/master/service/dynamodb/customizations.go
	minBackoff = 50 * time.Millisecond
	maxBackoff = 50 * time.Second
	maxRetries = 20

	// See http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html.
	dynamoMaxBatchSize = 25
)

var (
	dynamoRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "dynamo_request_duration_seconds",
		Help:      "Time spent doing DynamoDB requests.",

		// DynamoDB latency seems to range from a few ms to a few sec and is
		// important.  So use 8 buckets from 64us to 8s.
		Buckets: prometheus.ExponentialBuckets(0.000128, 4, 8),
	}, []string{"operation", "status_code"})
	dynamoConsumedCapacity = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "dynamo_consumed_capacity_total",
		Help:      "The capacity units consumed by operation.",
	}, []string{"operation"})
	dynamoFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "dynamo_failures_total",
		Help:      "The total number of errors while storing chunks to the chunk store.",
	}, []string{tableNameLabel, errorReasonLabel})
)

func init() {
	prometheus.MustRegister(dynamoRequestDuration)
	prometheus.MustRegister(dynamoConsumedCapacity)
	prometheus.MustRegister(dynamoFailures)
}

type dynamoClientAdapter struct {
	DynamoDB dynamodbiface.DynamoDBAPI
}

// NewDynamoDBClient makes a new DynamoDBClient
func NewDynamoDBClient(dynamoDBURL string) (StorageClient, string, error) {
	url, err := url.Parse(dynamoDBURL)
	if err != nil {
		return nil, "", err
	}

	dynamoDBConfig, err := awsConfigFromURL(url)
	if err != nil {
		return nil, "", err
	}

	dynamoDBClient := dynamoClientAdapter{dynamodb.New(session.New(dynamoDBConfig))}
	tableName := strings.TrimPrefix(url.Path, "/")
	return dynamoDBClient, tableName, nil
}

func (d dynamoClientAdapter) NewWriteBatch() WriteBatch {
	return dynamoDBWriteBatch(map[string][]*dynamodb.WriteRequest{})
}

// batchWrite writes requests to the underlying storage, handling retires and backoff.
func (d dynamoClientAdapter) BatchWrite(ctx context.Context, input WriteBatch) error {
	outstanding := input.(dynamoDBWriteBatch)
	unprocessed := map[string][]*dynamodb.WriteRequest{}
	backoff, numRetries := minBackoff, 0
	for dictLen(outstanding)+dictLen(unprocessed) > 0 && numRetries < maxRetries {
		reqs := map[string][]*dynamodb.WriteRequest{}
		takeReqs(unprocessed, reqs, dynamoMaxBatchSize)
		takeReqs(outstanding, reqs, dynamoMaxBatchSize)
		var resp *dynamodb.BatchWriteItemOutput

		err := instrument.TimeRequestHistogram(ctx, "DynamoDB.BatchWriteItem", dynamoRequestDuration, func(_ context.Context) error {
			var err error
			resp, err = d.DynamoDB.BatchWriteItem(&dynamodb.BatchWriteItemInput{
				RequestItems:           reqs,
				ReturnConsumedCapacity: aws.String(dynamodb.ReturnConsumedCapacityTotal),
			})
			return err
		})
		for _, cc := range resp.ConsumedCapacity {
			dynamoConsumedCapacity.WithLabelValues("DynamoDB.BatchWriteItem").
				Add(float64(*cc.CapacityUnits))
		}

		if err != nil {
			for tableName := range reqs {
				recordDynamoError(tableName, err)
			}
		}

		// If there are unprocessed items, backoff and retry those items.
		if unprocessedItems := resp.UnprocessedItems; unprocessedItems != nil && dictLen(unprocessedItems) > 0 {
			takeReqs(unprocessedItems, unprocessed, -1)
			time.Sleep(backoff)
			backoff = nextBackoff(backoff)
			continue
		}

		// If we get provisionedThroughputExceededException, then no items were processed,
		// so back off and retry all.
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == provisionedThroughputExceededException {
			takeReqs(reqs, unprocessed, -1)
			time.Sleep(backoff)
			backoff = nextBackoff(backoff)
			numRetries++
			continue
		}

		// All other errors are fatal.
		if err != nil {
			return err
		}

		backoff = minBackoff
		numRetries = 0
	}

	if valuesLeft := dictLen(outstanding) + dictLen(unprocessed); valuesLeft > 0 {
		return fmt.Errorf("failed to write chunk after %d retries, %d values remaining", numRetries, valuesLeft)
	}
	return nil
}

func (d dynamoClientAdapter) QueryPages(ctx context.Context, entry IndexEntry, callback func(result ReadBatch, lastPage bool) (shouldContinue bool)) error {
	input := &dynamodb.QueryInput{
		TableName: aws.String(entry.TableName),
		KeyConditions: map[string]*dynamodb.Condition{
			hashKey: {
				AttributeValueList: []*dynamodb.AttributeValue{
					{S: aws.String(entry.HashValue)},
				},
				ComparisonOperator: aws.String(dynamodb.ComparisonOperatorEq),
			},
		},
		ReturnConsumedCapacity: aws.String(dynamodb.ReturnConsumedCapacityTotal),
	}

	if entry.RangeValuePrefix != nil {
		input.KeyConditions[rangeKey] = &dynamodb.Condition{
			AttributeValueList: []*dynamodb.AttributeValue{
				{B: entry.RangeValuePrefix},
			},
			ComparisonOperator: aws.String(dynamodb.ComparisonOperatorBeginsWith),
		}
	} else if entry.RangeValueStart != nil {
		input.KeyConditions[rangeKey] = &dynamodb.Condition{
			AttributeValueList: []*dynamodb.AttributeValue{
				{B: entry.RangeValueStart},
			},
			ComparisonOperator: aws.String(dynamodb.ComparisonOperatorGe),
		}
	}

	request, _ := d.DynamoDB.QueryRequest(input)
	backoff := minBackoff
	for page := request; page != nil; page = page.NextPage() {
		err := instrument.TimeRequestHistogram(ctx, "DynamoDB.QueryPages", dynamoRequestDuration, func(_ context.Context) error {
			return page.Send()
		})

		if cc := page.Data.(*dynamodb.QueryOutput).ConsumedCapacity; cc != nil {
			dynamoConsumedCapacity.WithLabelValues("DynamoDB.QueryPages").
				Add(float64(*cc.CapacityUnits))
		}

		if err != nil {
			recordDynamoError(*input.TableName, err)

			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == provisionedThroughputExceededException {
				time.Sleep(backoff)
				backoff = nextBackoff(backoff)
				continue
			}

			return page.Error
		}

		queryOutput := page.Data.(*dynamodb.QueryOutput)
		if getNextPage := callback(dynamoDBReadBatch(queryOutput.Items), !page.HasNextPage()); !getNextPage {
			return page.Error
		}

		backoff = minBackoff
	}

	return nil
}

func (d dynamoClientAdapter) ListTables() ([]string, error) {
	table := []string{}
	if err := d.DynamoDB.ListTablesPages(&dynamodb.ListTablesInput{}, func(resp *dynamodb.ListTablesOutput, _ bool) bool {
		for _, s := range resp.TableNames {
			table = append(table, *s)
		}
		return true
	}); err != nil {
		return nil, err
	}
	return table, nil
}

func (d dynamoClientAdapter) CreateTable(name string, readCapacity, writeCapacity int64) error {
	input := &dynamodb.CreateTableInput{
		TableName: aws.String(name),
		AttributeDefinitions: []*dynamodb.AttributeDefinition{
			{
				AttributeName: aws.String(hashKey),
				AttributeType: aws.String(dynamodb.ScalarAttributeTypeS),
			},
			{
				AttributeName: aws.String(rangeKey),
				AttributeType: aws.String(dynamodb.ScalarAttributeTypeB),
			},
		},
		KeySchema: []*dynamodb.KeySchemaElement{
			{
				AttributeName: aws.String(hashKey),
				KeyType:       aws.String(dynamodb.KeyTypeHash),
			},
			{
				AttributeName: aws.String(rangeKey),
				KeyType:       aws.String(dynamodb.KeyTypeRange),
			},
		},
		ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
			ReadCapacityUnits:  aws.Int64(readCapacity),
			WriteCapacityUnits: aws.Int64(writeCapacity),
		},
	}
	_, err := d.DynamoDB.CreateTable(input)
	return err
}

func (d dynamoClientAdapter) DescribeTable(name string) (readCapacity, writeCapacity int64, status string, err error) {
	out, err := d.DynamoDB.DescribeTable(&dynamodb.DescribeTableInput{
		TableName: aws.String(name),
	})
	if err != nil {
		return 0, 0, "", err
	}

	return *out.Table.ProvisionedThroughput.ReadCapacityUnits, *out.Table.ProvisionedThroughput.WriteCapacityUnits, *out.Table.TableStatus, nil
}

func (d dynamoClientAdapter) UpdateTable(name string, readCapacity, writeCapacity int64) error {
	_, err := d.DynamoDB.UpdateTable(&dynamodb.UpdateTableInput{
		TableName: aws.String(name),
		ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
			ReadCapacityUnits:  aws.Int64(readCapacity),
			WriteCapacityUnits: aws.Int64(writeCapacity),
		},
	})
	return err
}

type dynamoDBWriteBatch map[string][]*dynamodb.WriteRequest

func (b dynamoDBWriteBatch) Add(tableName, hashValue string, rangeValue []byte) {
	b[tableName] = append(b[tableName], &dynamodb.WriteRequest{
		PutRequest: &dynamodb.PutRequest{
			Item: map[string]*dynamodb.AttributeValue{
				hashKey:  {S: aws.String(hashValue)},
				rangeKey: {B: rangeValue},
			},
		},
	})
}

type dynamoDBReadBatch []map[string]*dynamodb.AttributeValue

func (b dynamoDBReadBatch) Len() int {
	return len(b)
}

func (b dynamoDBReadBatch) RangeValue(i int) []byte {
	return b[i][rangeKey].B
}

func (b dynamoDBReadBatch) Value(i int) []byte {
	chunkValue, ok := b[i][chunkKey]
	if !ok {
		return nil
	}
	return chunkValue.B
}

func nextBackoff(lastBackoff time.Duration) time.Duration {
	// Based on the "Decorrelated Jitter" approach from https://www.awsarchitectureblog.com/2015/03/backoff.html
	// sleep = min(cap, random_between(base, sleep * 3))
	backoff := minBackoff + time.Duration(rand.Int63n(int64((lastBackoff*3)-minBackoff)))
	if backoff > maxBackoff {
		backoff = maxBackoff
	}
	return backoff
}

func recordDynamoError(tableName string, err error) {
	if awsErr, ok := err.(awserr.Error); ok {
		dynamoFailures.WithLabelValues(tableName, awsErr.Code()).Add(float64(1))
	} else {
		dynamoFailures.WithLabelValues(tableName, otherError).Add(float64(1))
	}
}

func dictLen(b map[string][]*dynamodb.WriteRequest) int {
	result := 0
	for _, reqs := range b {
		result += len(reqs)
	}
	return result
}

// Fill 'to' with WriteRequests from 'from' until 'to' has at most max requests. Remove those requests from 'from'.
func takeReqs(from, to map[string][]*dynamodb.WriteRequest, max int) {
	outLen, inLen := dictLen(to), dictLen(from)
	toFill := inLen
	if max > 0 {
		toFill = util.Min(inLen, max-outLen)
	}
	for toFill > 0 {
		for tableName, fromReqs := range from {
			taken := util.Min(len(fromReqs), toFill)
			if taken > 0 {
				to[tableName] = append(to[tableName], fromReqs[:taken]...)
				from[tableName] = fromReqs[taken:]
				toFill -= taken
			}
		}
	}
}
<|endoftext|>