<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A program that makes random reads within a large object on GCS, reporting
// throughput and latency information.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"sort"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/syncutil"
)

var fBucket = flag.String("bucket", "", "The GCS bucket from which to read.")
var fObject = flag.String("object", "", "The object within which to read.")
var fSize = flag.Uint64("size", 1<<21, "The size of each read in bytes.")
var fWorkers = flag.Int("workers", 1, "The number of workers to run.")

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type result struct {
	BytesRead int

	// Time taken from making the HTTP request to receiving the first byte of the
	// response body.
	FirstByteLatency time.Duration

	// Time taken from making the HTTP request to receiving the last byte of the
	// response body.
	FullBodyDuration time.Duration
}

func readOnce(
	ctx context.Context,
	o *gcs.Object,
	bucket gcs.Bucket) (r result, err error) {
	// Is the object large enough?
	if o.Size < *fSize {
		err = fmt.Errorf(
			"Object of size %d not large enough for read size %d",
			o.Size,
			*fSize)

		return
	}

	// Set up an appropriate request.
	req := &gcs.ReadObjectRequest{
		Name:       o.Name,
		Generation: o.Generation,
		Range:      &gcs.ByteRange{},
	}

	req.Range.Start = uint64(rand.Int63n(int64(o.Size - *fSize)))
	req.Range.Limit = req.Range.Start + *fSize

	// Create the reader.
	start := time.Now()
	rc, err := bucket.NewReader(ctx, req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	defer func() {
		closeErr := rc.Close()
		if err == nil && closeErr != nil {
			err = fmt.Errorf("Close: %v", closeErr)
		}
	}()

	// Measure the time to first byte.
	_, err = rc.Read([]byte{0})
	if err != nil {
		err = fmt.Errorf("Read: %v", err)
		return
	}

	r.FirstByteLatency = time.Since(start)

	// And the time to read everything.
	n, err := io.Copy(ioutil.Discard, rc)
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	r.FullBodyDuration = time.Since(start)
	r.BytesRead = int(n + 1)

	return
}

// Make random reads within the given object, writing results to the supplied
// channel. Stop when the stop channel is closed.
func makeReads(
	ctx context.Context,
	o *gcs.Object,
	bucket gcs.Bucket,
	results chan<- result,
	stop <-chan struct{}) (err error) {
	// Is the object large enough?
	if o.Size < *fSize {
		err = fmt.Errorf(
			"Object of size %d not large enough for read size %d",
			o.Size,
			*fSize)

		return
	}

	for {
		// Stop?
		select {
		case <-stop:
			return
		default:
		}

		// Make a read.
		var r result
		r, err = readOnce(ctx, o, bucket)
		if err != nil {
			err = fmt.Errorf("readOnce %v", err)
			return
		}

		log.Printf(
			"Read %d bytes in %v (first byte latency %v)",
			r.BytesRead,
			r.FullBodyDuration,
			r.FirstByteLatency)

		// Write out the result.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case results <- r:
		}
	}
}

func getBucket(ctx context.Context) (b gcs.Bucket, err error) {
	if *fBucket == "" {
		err = errors.New("You must set --bucket.")
		return
	}

	// Set up the token source.
	const scope = gcs.Scope_ReadOnly
	tokenSrc, err := google.DefaultTokenSource(context.Background(), scope)
	if err != nil {
		err = fmt.Errorf("DefaultTokenSource: %v", err)
		return
	}

	// Use that to create a GCS connection.
	cfg := &gcs.ConnConfig{
		TokenSource: tokenSrc,
	}

	conn, err := gcs.NewConn(cfg)
	if err != nil {
		err = fmt.Errorf("NewConn: %v", err)
		return
	}

	// Open the bucket.
	b, err = conn.OpenBucket(ctx, *fBucket)
	if err != nil {
		err = fmt.Errorf("OpenBucket: %v", err)
		return
	}

	return
}

func getObject(
	ctx context.Context,
	bucket gcs.Bucket) (o *gcs.Object, err error) {
	if *fObject == "" {
		err = errors.New("You must set --object.")
		return
	}

	o, err = bucket.StatObject(
		ctx,
		&gcs.StatObjectRequest{Name: *fObject})

	return
}

// Run workers until SIGINT is received. Return a slice of results.
func runWorkers(
	ctx context.Context,
	o *gcs.Object,
	bucket gcs.Bucket) (results []result, err error) {
	b := syncutil.NewBundle(ctx)

	// Set up a channel that is closed upon SIGINT.
	stop := make(chan struct{})
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)

		<-c
		log.Printf("SIGINT received. Stopping soon...")
		close(stop)
	}()

	// Start several workers making random reads.
	var wg sync.WaitGroup
	resultChan := make(chan result)
	for i := 0; i < *fWorkers; i++ {
		wg.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer wg.Done()
			err = makeReads(ctx, o, bucket, resultChan, stop)
			if err != nil {
				err = fmt.Errorf("makeReads: %v", err)
				return
			}

			return
		})
	}

	go func() {
		wg.Wait()
		close(resultChan)
	}()

	// Accumulate results.
	b.Add(func(ctx context.Context) (err error) {
		for r := range resultChan {
			results = append(results, r)
		}

		return
	})

	err = b.Join()
	return
}

// REQUIRES: len(vals) > 0
func percentile(
	vals sort.IntSlice,
	n int) (x int)

func describeResults(results []result) {
	fmt.Printf("Made %d reads.\n\n", len(results))

	ptiles := []int{0, 50, 95, 99, 100}

	// Compute first byte latency percentiles.
	var vals sort.IntSlice
	for _, r := range results {
		vals = append(vals, int(r.FirstByteLatency))
	}
	sort.Sort(vals)

	fmt.Printf("First byte latency stats:\n")
	for _, ptile := range ptiles {
		fmt.Printf(
			"  %2d ptile: %v\n", ptile,
			time.Duration(percentile(vals, ptile)))
	}

	// Compute overall duration percentiles.
	vals = nil
	for _, r := range results {
		vals = append(vals, int(r.FullBodyDuration))
	}
	sort.Sort(vals)

	fmt.Printf("Full body stats:\n")
	for _, ptile := range ptiles {
		d := time.Duration(percentile(vals, ptile))
		bw := float64(*fSize) / (float64(d) / float64(time.Second))

		fmt.Printf("  %2d ptile: %v\n (%f MB/s)", ptile, d, bw)
	}
}

////////////////////////////////////////////////////////////////////////
// Main
////////////////////////////////////////////////////////////////////////

func run(ctx context.Context) (err error) {
	// Open the bucket.
	bucket, err := getBucket(ctx)
	if err != nil {
		err = fmt.Errorf("getBucket: %v", err)
		return
	}

	// Stat the object.
	o, err := getObject(ctx, bucket)
	if err != nil {
		err = fmt.Errorf("getObject: %v", err)
		return
	}

	// Make reads.
	results, err := runWorkers(ctx, o, bucket)
	if err != nil {
		err = fmt.Errorf("runWorkers: %v", err)
		return
	}

	// Print information about the results.
	describeResults(results)

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds)
	flag.Parse()

	err := run(context.Background())
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>percentile<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A program that makes random reads within a large object on GCS, reporting
// throughput and latency information.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"sort"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/syncutil"
)

var fBucket = flag.String("bucket", "", "The GCS bucket from which to read.")
var fObject = flag.String("object", "", "The object within which to read.")
var fSize = flag.Uint64("size", 1<<21, "The size of each read in bytes.")
var fWorkers = flag.Int("workers", 1, "The number of workers to run.")

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type result struct {
	BytesRead int

	// Time taken from making the HTTP request to receiving the first byte of the
	// response body.
	FirstByteLatency time.Duration

	// Time taken from making the HTTP request to receiving the last byte of the
	// response body.
	FullBodyDuration time.Duration
}

func readOnce(
	ctx context.Context,
	o *gcs.Object,
	bucket gcs.Bucket) (r result, err error) {
	// Is the object large enough?
	if o.Size < *fSize {
		err = fmt.Errorf(
			"Object of size %d not large enough for read size %d",
			o.Size,
			*fSize)

		return
	}

	// Set up an appropriate request.
	req := &gcs.ReadObjectRequest{
		Name:       o.Name,
		Generation: o.Generation,
		Range:      &gcs.ByteRange{},
	}

	req.Range.Start = uint64(rand.Int63n(int64(o.Size - *fSize)))
	req.Range.Limit = req.Range.Start + *fSize

	// Create the reader.
	start := time.Now()
	rc, err := bucket.NewReader(ctx, req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	defer func() {
		closeErr := rc.Close()
		if err == nil && closeErr != nil {
			err = fmt.Errorf("Close: %v", closeErr)
		}
	}()

	// Measure the time to first byte.
	_, err = rc.Read([]byte{0})
	if err != nil {
		err = fmt.Errorf("Read: %v", err)
		return
	}

	r.FirstByteLatency = time.Since(start)

	// And the time to read everything.
	n, err := io.Copy(ioutil.Discard, rc)
	if err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	r.FullBodyDuration = time.Since(start)
	r.BytesRead = int(n + 1)

	return
}

// Make random reads within the given object, writing results to the supplied
// channel. Stop when the stop channel is closed.
func makeReads(
	ctx context.Context,
	o *gcs.Object,
	bucket gcs.Bucket,
	results chan<- result,
	stop <-chan struct{}) (err error) {
	// Is the object large enough?
	if o.Size < *fSize {
		err = fmt.Errorf(
			"Object of size %d not large enough for read size %d",
			o.Size,
			*fSize)

		return
	}

	for {
		// Stop?
		select {
		case <-stop:
			return
		default:
		}

		// Make a read.
		var r result
		r, err = readOnce(ctx, o, bucket)
		if err != nil {
			err = fmt.Errorf("readOnce %v", err)
			return
		}

		log.Printf(
			"Read %d bytes in %v (first byte latency %v)",
			r.BytesRead,
			r.FullBodyDuration,
			r.FirstByteLatency)

		// Write out the result.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case results <- r:
		}
	}
}

func getBucket(ctx context.Context) (b gcs.Bucket, err error) {
	if *fBucket == "" {
		err = errors.New("You must set --bucket.")
		return
	}

	// Set up the token source.
	const scope = gcs.Scope_ReadOnly
	tokenSrc, err := google.DefaultTokenSource(context.Background(), scope)
	if err != nil {
		err = fmt.Errorf("DefaultTokenSource: %v", err)
		return
	}

	// Use that to create a GCS connection.
	cfg := &gcs.ConnConfig{
		TokenSource: tokenSrc,
	}

	conn, err := gcs.NewConn(cfg)
	if err != nil {
		err = fmt.Errorf("NewConn: %v", err)
		return
	}

	// Open the bucket.
	b, err = conn.OpenBucket(ctx, *fBucket)
	if err != nil {
		err = fmt.Errorf("OpenBucket: %v", err)
		return
	}

	return
}

func getObject(
	ctx context.Context,
	bucket gcs.Bucket) (o *gcs.Object, err error) {
	if *fObject == "" {
		err = errors.New("You must set --object.")
		return
	}

	o, err = bucket.StatObject(
		ctx,
		&gcs.StatObjectRequest{Name: *fObject})

	return
}

// Run workers until SIGINT is received. Return a slice of results.
func runWorkers(
	ctx context.Context,
	o *gcs.Object,
	bucket gcs.Bucket) (results []result, err error) {
	b := syncutil.NewBundle(ctx)

	// Set up a channel that is closed upon SIGINT.
	stop := make(chan struct{})
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)

		<-c
		log.Printf("SIGINT received. Stopping soon...")
		close(stop)
	}()

	// Start several workers making random reads.
	var wg sync.WaitGroup
	resultChan := make(chan result)
	for i := 0; i < *fWorkers; i++ {
		wg.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer wg.Done()
			err = makeReads(ctx, o, bucket, resultChan, stop)
			if err != nil {
				err = fmt.Errorf("makeReads: %v", err)
				return
			}

			return
		})
	}

	go func() {
		wg.Wait()
		close(resultChan)
	}()

	// Accumulate results.
	b.Add(func(ctx context.Context) (err error) {
		for r := range resultChan {
			results = append(results, r)
		}

		return
	})

	err = b.Join()
	return
}

// REQUIRES: len(vals) > 0
// REQUIRES: 0 <= n <= 100
func percentile(
	vals sort.IntSlice,
	n int) (x int) {
	// Special cases.
	switch {
	case n == 0:
		x = vals[0]
		return

	case n == 100:
		x = vals[len(vals)-1]
		return
	}

	// Find the nearest, truncating (why not).
	index := int((float64(n) / 100) * float64(len(vals)))
	x = vals[index]

	return
}

func describeResults(results []result) {
	fmt.Printf("Made %d reads.\n\n", len(results))

	ptiles := []int{0, 50, 95, 99, 100}

	// Compute first byte latency percentiles.
	var vals sort.IntSlice
	for _, r := range results {
		vals = append(vals, int(r.FirstByteLatency))
	}
	sort.Sort(vals)

	fmt.Printf("First byte latency stats:\n")
	for _, ptile := range ptiles {
		fmt.Printf(
			"  %2d ptile: %v\n", ptile,
			time.Duration(percentile(vals, ptile)))
	}

	// Compute overall duration percentiles.
	vals = nil
	for _, r := range results {
		vals = append(vals, int(r.FullBodyDuration))
	}
	sort.Sort(vals)

	fmt.Printf("Full body stats:\n")
	for _, ptile := range ptiles {
		d := time.Duration(percentile(vals, ptile))
		bw := float64(*fSize) / (float64(d) / float64(time.Second))

		fmt.Printf("  %2d ptile: %v\n (%f MB/s)", ptile, d, bw)
	}
}

////////////////////////////////////////////////////////////////////////
// Main
////////////////////////////////////////////////////////////////////////

func run(ctx context.Context) (err error) {
	// Open the bucket.
	bucket, err := getBucket(ctx)
	if err != nil {
		err = fmt.Errorf("getBucket: %v", err)
		return
	}

	// Stat the object.
	o, err := getObject(ctx, bucket)
	if err != nil {
		err = fmt.Errorf("getObject: %v", err)
		return
	}

	// Make reads.
	results, err := runWorkers(ctx, o, bucket)
	if err != nil {
		err = fmt.Errorf("runWorkers: %v", err)
		return
	}

	// Print information about the results.
	describeResults(results)

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds)
	flag.Parse()

	err := run(context.Background())
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// The MIT License (MIT)

// Copyright (c) 2016 Maciej Borzecki

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package models

import (
	"github.com/jinzhu/gorm"
	"time"
)

// Match data
type Match struct {
	gorm.Model

	// match data hash
	DataHash string `sql:"not null,unique"`
	// when the match was played
	DateTime time.Time
	// duration in seconds
	Duration uint
	// map, ex. q3dm17
	Map string
	// type of match, 1v1, FFA, CTF
	Type string
}

func (m Match) DurationDesc() string {
	return (time.Duration(m.Duration) * time.Second).String()
}
<commit_msg>models/match: helper for nice printing of match date/time<commit_after>// The MIT License (MIT)

// Copyright (c) 2016 Maciej Borzecki

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package models

import (
	"github.com/jinzhu/gorm"
	"time"
)

// Match data
type Match struct {
	gorm.Model

	// match data hash
	DataHash string `sql:"not null,unique"`
	// when the match was played
	DateTime time.Time
	// duration in seconds
	Duration uint
	// map, ex. q3dm17
	Map string
	// type of match, 1v1, FFA, CTF
	Type string
}

func (m Match) DurationDesc() string {
	return (time.Duration(m.Duration) * time.Second).String()
}

func (m Match) NiceDateTime() string {
	return m.DateTime.Format("02-01-2006 15:04:05")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var (
	environment string
	configPath string
	dbURL string
	herokuApp string
	herokuEnvVar = "DATABASE_URL"
)

type Config map[string]interface{}
type Configs map[string]Config

// Pull environment from:
// 1. explicitly-set or previously-cached string
// 2. $ENV environment variable
// 3. $RAILS_ENV environment variable
// 4. default: "development"
func env() string {
	if len(environment) != 0 {
		return environment
	}
	environment = os.Getenv("ENV")
	if len(environment) != 0 {
		return environment
	}
	environment = os.Getenv("RAILS_ENV")
	if len(environment) != 0 {
		return environment
	}
	return "development"
}

// Pull config from:
// 1. explicitly-set string
// 2. config/database.yml
func path() string {
	if len(configPath) != 0 {
		return configPath
	}
	return "config/database.yml"
}

// Pull database URL from:
// 1. explicitly-set string
// 2. Heroku app config var
// 3. $DATABASE_URL environment variable
func uri() string {
	if len(dbURL) != 0 {
		return dbURL
	}

	if len(herokuApp) != 0 {
		return fetchHerokuConfig()
	}

	dbURL = os.Getenv("DATABASE_URL")
	return dbURL
}

func fetchHerokuConfig() string {
	args := []string{"config:get", herokuEnvVar, "-a", herokuApp}
	out, err := exec.Command("heroku", args...).Output()
	if err == nil {
		return string(out[0:len(out)-1]) // strip trailing newline
	}
	return ""
}

// Only Ruby can parse embedded Ruby (ERB)
func parseERB(in []byte) []byte {
	if !strings.Contains(string(in), "<%") {
		return in
	}

	var err error
	// consolidate tedious error handling
	errHandle := func() {
		if err != nil {
			log.Panic(err)
		}
	}

	cmd := exec.Command("ruby", "-rerb", "-e", "puts ERB.new(ARGF.read).result")
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Panic(err)
	}
	stdout, err := cmd.StdoutPipe()
	errHandle()
	err = cmd.Start()
	errHandle()
	stdin.Write(in)
	stdin.Close()
	out, err := ioutil.ReadAll(stdout)
	errHandle()
	err = cmd.Wait()
	errHandle()
	return out
}

// Read the file, parse out the ERB (if needed), and return the mappings
func parseDatabaseConfig(filename string) Configs {
	var parsed Configs
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Panic(err)
	}
	withoutERB := parseERB(body)
	err = yaml.Unmarshal(withoutERB, &parsed)
	if err != nil {
		log.Panic(err)
	}
	return parsed
}

// Read the database config and pull out the specified environment's settings
func fromPath(filename, env string) Config {
	configs := parseDatabaseConfig(filename)
	if config, found := configs[env]; found {
		return config
	}
	return nil
}

func fromURL(uri string) Config {
	parsed, err := url.Parse(uri)
	if err != nil {
		log.Panic(err)
	}

	ret := Config{}

	if len(parsed.Scheme) > 0 {
		ret["adapter"] = parsed.Scheme
	}

	if len(parsed.Host) > 0 {
		hostPort := strings.Split(parsed.Host, ":")
		switch len(hostPort) {
		case 1:
			ret["host"] = hostPort[0]
		case 2:
			port, err := strconv.ParseInt(hostPort[1], 0, 0)
			if err == nil {
				ret["host"] = hostPort[0]
				ret["port"] = port
			} else {
				ret["host"] = parsed.Host
			}
		default:
			ret["host"] = parsed.Host
		}
	}

	if len(parsed.Path) > 1 {
		// remove leading slash
		ret["database"] = parsed.Path[1:len(parsed.Path)]
	}

	if parsed.User != nil {
		ret["username"] = parsed.User.Username()
		pass, present := parsed.User.Password()
		if present {
			ret["password"] = pass
		}
	}

	for k, strs := range(parsed.Query()) {
		if len(strs) != 1 {
			ret[k] = strs
		} else {
			s := strs[0]
			num, err := strconv.ParseInt(s, 0, 0)
			if err == nil {
				ret[k] = num
			} else {
				ret[k] = s
			}
		}
	}

	return ret
}

func mappedAdapter(adapter string) string {
	switch adapter {
	case "pg", "postgresql":
		return "postgres"
	default:
		return adapter
	}
}

func (config Config) ToURL() *url.URL {
	ret := &url.URL{}
	hostPort := []string{}
	user, pass := "", ""
	hasUser, hasPass := false, false
	params := &url.Values{}
	hasParams := false

	for k, v := range(config) {
		var val string
		switch t := v.(type) {
		case string:
			val = t
		case interface{String() string}:
			val = t.String()
		case nil:
			val = ""
		default:
			val = fmt.Sprintf("%v", v)
		}

		switch k {
		case "adapter":
			ret.Scheme = mappedAdapter(val)
		case "host", "hostname":
			hostPort = append([]string{val}, hostPort...)
		case "port":
			hostPort = append(hostPort, val)
		case "database":
			ret.Path = fmt.Sprintf("/%s", val)
		case "user", "username":
			user = val
			hasUser = true
		case "pass", "password":
			pass = val
			hasPass = true
		default:
			params.Add(k, val)
			hasParams = true
		}
	}

	if len(hostPort) != 0 {
		ret.Host = strings.Join(hostPort, ":")
	}

	if hasPass {
		ret.User = url.UserPassword(user, pass)
	} else if hasUser {
		ret.User = url.User(user)
	}

	if hasParams {
		ret.RawQuery = params.Encode()
	}

	return ret
}

func inputConfig(useURL, useYAML bool) Config {
	if useURL {
		return fromURL(uri())
	}
	return fromPath(path(), env())
}

func outputConfig(cfg Config, asURL, asYAML bool) {
	if asYAML {
		yml, err := yaml.Marshal(&Configs{env(): cfg})
		if err != nil {
			log.Panic(err)
		}
		os.Stdout.Write(yml)
	} else {
		fmt.Println(cfg.ToURL().String())
	}
}

func (c Config) Transform(name string) {
	if len(name) > 0 {
		c["database"] = name
	}
}

func main() {
	// Input options
	parseYAML := false
	parseURL := false

	flag.BoolVar(&parseYAML, "fromyaml", parseYAML, "Parse starting config from YAML")
	flag.BoolVar(&parseURL, "fromurl", parseURL, "Parse starting config from URL")
	flag.StringVar(&herokuApp, "fromheroku", herokuApp, "Heroku app name")
	flag.StringVar(&herokuEnvVar, "herokuvar", herokuEnvVar, "Heroku environment variable name")

	// Output options
	emitURL := false
	emitYAML := false

	flag.BoolVar(&emitURL, "url", emitURL, "Output a URL")
	flag.BoolVar(&emitYAML, "yaml", emitYAML, "Output YAML")

	// Selection options
	flag.StringVar(&environment, "env", environment, "Environment name")
	flag.StringVar(&configPath, "path", configPath, "database.yml file to parse")
	flag.StringVar(&dbURL, "dburl", dbURL, "database URL to parse")

	// Transformation options
	dbName := ""

	flag.StringVar(&dbName, "dbname", dbName, "Use this as the database name")

	flag.Parse()

	if len(herokuApp) != 0 {
		parseURL = true
		if len(environment) == 0 {
			environment = herokuApp
		}
	}

	cfg := inputConfig(parseURL, parseYAML)
	cfg.Transform(dbName)
	outputConfig(cfg, emitURL, emitYAML)
}
<commit_msg>Allow passing args to `db-url` as a command to run<commit_after>package main

import (
	"flag"
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
)

var (
	environment string
	envVar = "DATABASE_URL"
	configPath string
	dbURL string
	herokuApp string
	herokuEnvVar = "DATABASE_URL"
)

type Config map[string]interface{}
type Configs map[string]Config

// Pull environment from:
// 1. explicitly-set or previously-cached string
// 2. $ENV environment variable
// 3. $RAILS_ENV environment variable
// 4. default: "development"
func env() string {
	if len(environment) != 0 {
		return environment
	}
	environment = os.Getenv("ENV")
	if len(environment) != 0 {
		return environment
	}
	environment = os.Getenv("RAILS_ENV")
	if len(environment) != 0 {
		return environment
	}
	return "development"
}

// Pull config from:
// 1. explicitly-set string
// 2. config/database.yml
func path() string {
	if len(configPath) != 0 {
		return configPath
	}
	return "config/database.yml"
}

// Pull database URL from:
// 1. explicitly-set string
// 2. Heroku app config var
// 3. $DATABASE_URL environment variable
func uri() string {
	if len(dbURL) != 0 {
		return dbURL
	}

	if len(herokuApp) != 0 {
		return fetchHerokuConfig()
	}

	dbURL = os.Getenv(envVar)
	return dbURL
}

func fetchHerokuConfig() string {
	args := []string{"config:get", herokuEnvVar, "-a", herokuApp}
	out, err := exec.Command("heroku", args...).Output()
	if err == nil {
		return string(out[0:len(out)-1]) // strip trailing newline
	}
	return ""
}

// Only Ruby can parse embedded Ruby (ERB)
func parseERB(in []byte) []byte {
	if !strings.Contains(string(in), "<%") {
		return in
	}

	var err error
	// consolidate tedious error handling
	errHandle := func() {
		if err != nil {
			log.Panic(err)
		}
	}

	cmd := exec.Command("ruby", "-rerb", "-e", "puts ERB.new(ARGF.read).result")
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Panic(err)
	}
	stdout, err := cmd.StdoutPipe()
	errHandle()
	err = cmd.Start()
	errHandle()
	stdin.Write(in)
	stdin.Close()
	out, err := ioutil.ReadAll(stdout)
	errHandle()
	err = cmd.Wait()
	errHandle()
	return out
}

// Read the file, parse out the ERB (if needed), and return the mappings
func parseDatabaseConfig(filename string) Configs {
	var parsed Configs
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Panic(err)
	}
	withoutERB := parseERB(body)
	err = yaml.Unmarshal(withoutERB, &parsed)
	if err != nil {
		log.Panic(err)
	}
	return parsed
}

// Read the database config and pull out the specified environment's settings
func fromPath(filename, env string) Config {
	configs := parseDatabaseConfig(filename)
	if config, found := configs[env]; found {
		return config
	}
	return nil
}

func fromURL(uri string) Config {
	parsed, err := url.Parse(uri)
	if err != nil {
		log.Panic(err)
	}

	ret := Config{}

	if len(parsed.Scheme) > 0 {
		ret["adapter"] = parsed.Scheme
	}

	if len(parsed.Host) > 0 {
		hostPort := strings.Split(parsed.Host, ":")
		switch len(hostPort) {
		case 1:
			ret["host"] = hostPort[0]
		case 2:
			port, err := strconv.ParseInt(hostPort[1], 0, 0)
			if err == nil {
				ret["host"] = hostPort[0]
				ret["port"] = port
			} else {
				ret["host"] = parsed.Host
			}
		default:
			ret["host"] = parsed.Host
		}
	}

	if len(parsed.Path) > 1 {
		// remove leading slash
		ret["database"] = parsed.Path[1:len(parsed.Path)]
	}

	if parsed.User != nil {
		ret["username"] = parsed.User.Username()
		pass, present := parsed.User.Password()
		if present {
			ret["password"] = pass
		}
	}

	for k, strs := range(parsed.Query()) {
		if len(strs) != 1 {
			ret[k] = strs
		} else {
			s := strs[0]
			num, err := strconv.ParseInt(s, 0, 0)
			if err == nil {
				ret[k] = num
			} else {
				ret[k] = s
			}
		}
	}

	return ret
}

func mappedAdapter(adapter string) string {
	switch adapter {
	case "pg", "postgresql":
		return "postgres"
	default:
		return adapter
	}
}

func (config Config) ToURL() *url.URL {
	ret := &url.URL{}
	hostPort := []string{}
	user, pass := "", ""
	hasUser, hasPass := false, false
	params := &url.Values{}
	hasParams := false

	for k, v := range(config) {
		var val string
		switch t := v.(type) {
		case string:
			val = t
		case interface{String() string}:
			val = t.String()
		case nil:
			val = ""
		default:
			val = fmt.Sprintf("%v", v)
		}

		switch k {
		case "adapter":
			ret.Scheme = mappedAdapter(val)
		case "host", "hostname":
			hostPort = append([]string{val}, hostPort...)
		case "port":
			hostPort = append(hostPort, val)
		case "database":
			ret.Path = fmt.Sprintf("/%s", val)
		case "user", "username":
			user = val
			hasUser = true
		case "pass", "password":
			pass = val
			hasPass = true
		default:
			params.Add(k, val)
			hasParams = true
		}
	}

	if len(hostPort) != 0 {
		ret.Host = strings.Join(hostPort, ":")
	}

	if hasPass {
		ret.User = url.UserPassword(user, pass)
	} else if hasUser {
		ret.User = url.User(user)
	}

	if hasParams {
		ret.RawQuery = params.Encode()
	}

	return ret
}

func inputConfig(useURL, useYAML bool) Config {
	if useURL {
		return fromURL(uri())
	}
	return fromPath(path(), env())
}

func outputConfig(cfg Config, asURL, asYAML bool) {
	if asYAML {
		yml, err := yaml.Marshal(&Configs{env(): cfg})
		if err != nil {
			log.Panic(err)
		}
		os.Stdout.Write(yml)
	} else {
		fmt.Println(cfg.ToURL().String())
	}
}

func (c Config) Transform(name string) {
	if len(name) > 0 {
		c["database"] = name
	}
}

func execWithEnv(cfg Config, args []string) {
	cmd := args[0]
	argv0, err := exec.LookPath(cmd)
	if err != nil {
		log.Fatalf("Couldn't find %s in PATH", cmd)
	}

	env := []string{}

	for _, nameVal := range os.Environ() {
		if !strings.HasPrefix(nameVal, fmt.Sprintf("%s=", envVar)) {
			env = append(env, nameVal)
		}
	}

	env = append(env, fmt.Sprintf("%s=%s", envVar, cfg.ToURL().String()))

	log.Fatal(syscall.Exec(argv0, args, env))
}

func main() {
	// Input options
	parseYAML := false
	parseURL := false

	flag.BoolVar(&parseYAML, "fromyaml", parseYAML, "Parse starting config from YAML")
	flag.BoolVar(&parseURL, "fromurl", parseURL, "Parse starting config from URL")
	flag.StringVar(&herokuApp, "fromheroku", herokuApp, "Heroku app name")
	flag.StringVar(&herokuEnvVar, "herokuvar", herokuEnvVar, "Heroku environment variable name")

	// Output options
	emitURL := false
	emitYAML := false

	flag.BoolVar(&emitURL, "url", emitURL, "Output a URL")
	flag.BoolVar(&emitYAML, "yaml", emitYAML, "Output YAML")

	// Selection options
	flag.StringVar(&environment, "env", environment, "Environment name")
	flag.StringVar(&envVar, "var", envVar, "Environment variable name")
	flag.StringVar(&configPath, "path", configPath, "database.yml file to parse")
	flag.StringVar(&dbURL, "dburl", dbURL, "database URL to parse")

	// Transformation options
	dbName := ""

	flag.StringVar(&dbName, "dbname", dbName, "Use this as the database name")

	flag.Parse()

	if len(herokuApp) != 0 {
		parseURL = true
		if len(environment) == 0 {
			environment = herokuApp
		}
	}

	cfg := inputConfig(parseURL, parseYAML)
	cfg.Transform(dbName)

	if len(flag.Args()) > 0 {
		execWithEnv(cfg, flag.Args())
	} else {
		outputConfig(cfg, emitURL, emitYAML)
	}
}
<|endoftext|>


<commit_before>// run

// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// windows doesn't work, because Windows exception handling
// delivers signals based on the current PC, and that current PC
// doesn't go into the Go runtime.
// +build !windows

// wasm does not work, because the linear memory is not executable.
// +build !wasm

// This test doesn't work on gccgo/GoLLVM, because they will not find
// any unwind information for the artificial function, and will not be
// able to unwind past that point.
// +build !gccgo

package main

import (
	"encoding/binary"
	"runtime"
	"runtime/debug"
	"unsafe"
)

func main() {
	debug.SetPanicOnFault(true)
	defer func() {
		if err := recover(); err == nil {
			panic("not panicking")
		}
		pc, _, _, _ := runtime.Caller(10)
		f := runtime.FuncForPC(pc)
		if f == nil || f.Name() != "main.f" {
			if f == nil {
				println("no func for ", unsafe.Pointer(pc))
			} else {
				println("found func:", f.Name())
			}
			panic("cannot find main.f on stack")
		}
	}()
	f(20)
}

func f(n int) {
	if n > 0 {
		f(n - 1)
	}
	var f struct {
		x uintptr
	}

	// We want to force a seg fault, to get a crash at a PC value != 0.
	// Not all systems make the data section non-executable.
	ill := make([]byte, 64)
	switch runtime.GOARCH {
	case "386", "amd64":
		ill = append(ill, 0x89, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00) // MOVL AX, 0
	case "arm":
		binary.LittleEndian.PutUint32(ill, 0xe3a00000) // MOVW $0, R0
		binary.LittleEndian.PutUint32(ill, 0xe5800000) // MOVW R0, (R0)
	case "arm64":
		binary.LittleEndian.PutUint32(ill, 0xf90003ff) // MOVD ZR, (ZR)
	case "ppc64":
		binary.BigEndian.PutUint32(ill, 0xf8000000) // MOVD R0, (R0)
	case "ppc64le":
		binary.LittleEndian.PutUint32(ill, 0xf8000000) // MOVD R0, (R0)
	case "mips", "mips64":
		binary.BigEndian.PutUint32(ill, 0xfc000000) // MOVV R0, (R0)
	case "mipsle", "mips64le":
		binary.LittleEndian.PutUint32(ill, 0xfc000000) // MOVV R0, (R0)
	case "s390x":
		ill = append(ill, 0xa7, 0x09, 0x00, 0x00)             // MOVD $0, R0
		ill = append(ill, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x24) // MOVD R0, (R0)
	case "riscv64":
		binary.LittleEndian.PutUint32(ill, 0x00003023) // MOV X0, (X0)
	default:
		// Just leave it as 0 and hope for the best.
	}

	f.x = uintptr(unsafe.Pointer(&ill[0]))
	p := &f
	fn := *(*func())(unsafe.Pointer(&p))
	fn()
}
<commit_msg>test: fix incorrectly laid out instructions in issue11656.go<commit_after>// run

// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// windows doesn't work, because Windows exception handling
// delivers signals based on the current PC, and that current PC
// doesn't go into the Go runtime.
// +build !windows

// wasm does not work, because the linear memory is not executable.
// +build !wasm

// This test doesn't work on gccgo/GoLLVM, because they will not find
// any unwind information for the artificial function, and will not be
// able to unwind past that point.
// +build !gccgo

package main

import (
	"encoding/binary"
	"runtime"
	"runtime/debug"
	"unsafe"
)

func main() {
	debug.SetPanicOnFault(true)
	defer func() {
		if err := recover(); err == nil {
			panic("not panicking")
		}
		pc, _, _, _ := runtime.Caller(10)
		f := runtime.FuncForPC(pc)
		if f == nil || f.Name() != "main.f" {
			if f == nil {
				println("no func for ", unsafe.Pointer(pc))
			} else {
				println("found func:", f.Name())
			}
			panic("cannot find main.f on stack")
		}
	}()
	f(20)
}

func f(n int) {
	if n > 0 {
		f(n - 1)
	}
	var f struct {
		x uintptr
	}

	// We want to force a seg fault, to get a crash at a PC value != 0.
	// Not all systems make the data section non-executable.
	ill := make([]byte, 64)
	switch runtime.GOARCH {
	case "386", "amd64":
		ill = append(ill[:0], 0x89, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00) // MOVL AX, 0
	case "arm":
		binary.LittleEndian.PutUint32(ill[0:4], 0xe3a00000) // MOVW $0, R0
		binary.LittleEndian.PutUint32(ill[4:8], 0xe5800000) // MOVW R0, (R0)
	case "arm64":
		binary.LittleEndian.PutUint32(ill, 0xf90003ff) // MOVD ZR, (ZR)
	case "ppc64":
		binary.BigEndian.PutUint32(ill, 0xf8000000) // MOVD R0, (R0)
	case "ppc64le":
		binary.LittleEndian.PutUint32(ill, 0xf8000000) // MOVD R0, (R0)
	case "mips", "mips64":
		binary.BigEndian.PutUint32(ill, 0xfc000000) // MOVV R0, (R0)
	case "mipsle", "mips64le":
		binary.LittleEndian.PutUint32(ill, 0xfc000000) // MOVV R0, (R0)
	case "s390x":
		ill = append(ill[:0], 0xa7, 0x09, 0x00, 0x00)         // MOVD $0, R0
		ill = append(ill, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x24) // MOVD R0, (R0)
	case "riscv64":
		binary.LittleEndian.PutUint32(ill, 0x00003023) // MOV X0, (X0)
	default:
		// Just leave it as 0 and hope for the best.
	}

	f.x = uintptr(unsafe.Pointer(&ill[0]))
	p := &f
	fn := *(*func())(unsafe.Pointer(&p))
	fn()
}
<|endoftext|>


<commit_before>/*Blacksheepwall is a hostname reconnaissance tool, it is similar to other
tools, but has a focus on speed.*/

package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sort"
	"text/tabwriter"

	"github.com/tomsteele/blacksheepwall/bsw"
)

const usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -timeout              Maximum timeout in seconds for SOCKET connections.  [default .5 seconds]
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6                 Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks, can be a
                        single domain or a file of line separated domains.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain.
  -bing <string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -srv                  Find DNS SRV record and retrieve associated hostname/IP info.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Processes a list of IP addresses or networks in CIDR format.
// Returning a list of all possible IP addresses.
func linesToIPList(lines []string) ([]string, error) {
	ipList := []string{}
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIP(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases an IP by a single address.
func increaseIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and return as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

type task func() (string, bsw.Results, error)
type empty struct{}

func main() {
	// Command line options. For usage information see the
	// usage variable above.
	var (
		flVersion     = flag.Bool("version", false, "")
		flTimeout     = flag.Int64("timeout", 600, "")
		flConcurrency = flag.Int("concurrency", 100, "")
		flDebug       = flag.Bool("debug", false, "")
		flipv6        = flag.Bool("ipv6", false, "")
		flServerAddr  = flag.String("server", "8.8.8.8", "")
		flIPFile      = flag.String("input", "", "")
		flReverse     = flag.Bool("reverse", false, "")
		flHeader      = flag.Bool("headers", false, "")
		flTLS         = flag.Bool("tls", false, "")
		flViewDNSInfo = flag.Bool("viewdns", false, "")
		flSRV         = flag.Bool("srv", false, "")
		flBing        = flag.String("bing", "", "")
		flYandex      = flag.String("yandex", "", "")
		flDomain      = flag.String("domain", "", "")
		flDictFile    = flag.String("dictionary", "", "")
		flFcrdns      = flag.Bool("fcrdns", false, "")
		flClean       = flag.Bool("clean", false, "")
		flCsv         = flag.Bool("csv", false, "")
		flJSON        = flag.Bool("json", false, "")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	// Modify timeout to Milliseconds for function calls
	if *flTimeout != 600 {
		*flTimeout = *flTimeout * 1000
	}

	// Holds all IP addresses for testing.
	ipAddrList := []string{}

	// Used to hold a ip or CIDR range passed as fl.Arg(0).
	var flNetwork string

	// Verify that some sort of work load was given in commands.
	if *flIPFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain == "" && *flSRV == true {
		log.Fatal("SRV lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" && *flSRV == false {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Build list of domains.
	domains := []string{}
	if *flDomain != "" {
		if _, err := os.Stat(*flDomain); os.IsNotExist(err) {
			domains = append(domains, *flDomain)
		} else {
			lines, err := readFileLines(*flDomain)
			if err != nil {
				log.Fatal("Error reading " + *flDomain + " " + err.Error())
			}
			domains = append(domains, lines...)
		}
	}

	// Get first argument that is not an option and turn it into a list of IPs.
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIPList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible IP or network into
	// a list of IPs. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid IP or CIDR range.
	if *flIPFile != "" {
		lines, err := readFileLines(*flIPFile)
		if err != nil {
			log.Fatal("Error reading " + *flIPFile + " " + err.Error())
		}
		list, err := linesToIPList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	// Use a map that acts like a set to store only unique results.
	resMap := make(map[bsw.Result]bool)

	// Start up *flConcurrency amount of goroutines.
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				task, result, err := def()
				if *flDebug == false {
					if m := c % 2; m == 0 {
						c = 3
						os.Stderr.WriteString("\rWorking \\")
					} else {
						c = 2
						os.Stderr.WriteString("\rWorking /")
					}
				}
				if err != nil && *flDebug {
					log.Printf("%v: %v", task, err.Error())
				}
				if err == nil {
					if *flDebug == true && len(result) > 0 {
						log.Printf("%v: %v %v: task completed successfully\n", task, result[0].Hostname, result[0].IP)
					}
					res <- result
				}
			}
			tracker <- empty{}
		}()
	}

	// Ingest incoming results.
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					} else {
						cfqdn, err := bsw.LookupCname(r.Hostname, *flServerAddr)
						if err == nil && len(cfqdn) > 0 {
							ip, err = bsw.LookupName(cfqdn, *flServerAddr)
							if err == nil && len(ip) > 0 {
								resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
							}
						}
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					}
				}
			} else {
				for _, r := range result {
					resMap[r] = true
				}
			}
		}
		tracker <- empty{}
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// IP based functionality should be added to the pool here.
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (string, bsw.Results, error) { return bsw.Reverse(host, *flServerAddr) }
		}
		if *flTLS {
			tasks <- func() (string, bsw.Results, error) { return bsw.TLS(host, *flTimeout) }
		}
		if *flViewDNSInfo {
			tasks <- func() (string, bsw.Results, error) { return bsw.ViewDNSInfo(host) }
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (string, bsw.Results, error) { return bsw.BingAPI(host, *flBing, bingPath) }
		}
		if *flHeader {
			tasks <- func() (string, bsw.Results, error) { return bsw.Headers(host, *flTimeout) }
		}
	}

	// Domain based functions will likely require separate blocks and should be added below.

	// Subdomain dictionary guessing.
	for _, d := range domains {
		domain := d
		if *flDictFile != "" {
			nameList, err := readFileLines(*flDictFile)
			if err != nil {
				log.Fatal("Error reading " + *flDictFile + " " + err.Error())
			}
			// Get an IP for a possible wildcard domain and use it as a blacklist.
			blacklist := bsw.GetWildCard(domain, *flServerAddr)
			var blacklist6 string
			if *flipv6 {
				blacklist6 = bsw.GetWildCard6(domain, *flServerAddr)
			}
			for _, n := range nameList {
				sub := n
				tasks <- func() (string, bsw.Results, error) { return bsw.Dictionary(domain, sub, blacklist, *flServerAddr) }
				if *flipv6 {
					tasks <- func() (string, bsw.Results, error) { return bsw.Dictionary6(domain, sub, blacklist6, *flServerAddr) }
				}
			}
		}

		if *flSRV != false {
			tasks <- func() (string, bsw.Results, error) { return bsw.SRV(domain, *flServerAddr) }
		}

		if *flYandex != "" {
			tasks <- func() (string, bsw.Results, error) { return bsw.YandexAPI(domain, *flYandex, *flServerAddr) }
		}
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer.
	<-tracker
	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")

	// Create a results slice from the unique set in resMap. Allows for sorting.
	results := bsw.Results{}
	for k := range resMap {
		results = append(results, k)
	}
	sort.Sort(results)

	switch {
	case *flJSON:
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	case *flCsv:
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	case *flClean:
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.IP] = append(cleanSet[r.IP], r.Hostname)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, h := range v {
				fmt.Printf("\t%s\n", h)
			}
		}
	default:
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<commit_msg>Minor style change<commit_after>/*Blacksheepwall is a hostname reconnaissance tool, it is similar to other
tools, but has a focus on speed.*/

package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sort"
	"text/tabwriter"

	"github.com/tomsteele/blacksheepwall/bsw"
)

const usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -timeout              Maximum timeout in seconds for SOCKET connections.  [default .5 seconds]
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6                 Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks, can be a
                        single domain or a file of line separated domains.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain.
  -bing <string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -srv                  Find DNS SRV record and retrieve associated hostname/IP info.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Processes a list of IP addresses or networks in CIDR format.
// Returning a list of all possible IP addresses.
func linesToIPList(lines []string) ([]string, error) {
	ipList := []string{}
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIP(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases an IP by a single address.
func increaseIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and return as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

type task func() (string, bsw.Results, error)
type empty struct{}

func main() {
	// Command line options. For usage information see the
	// usage variable above.
	var (
		flVersion     = flag.Bool("version", false, "")
		flTimeout     = flag.Int64("timeout", 600, "")
		flConcurrency = flag.Int("concurrency", 100, "")
		flDebug       = flag.Bool("debug", false, "")
		flipv6        = flag.Bool("ipv6", false, "")
		flServerAddr  = flag.String("server", "8.8.8.8", "")
		flIPFile      = flag.String("input", "", "")
		flReverse     = flag.Bool("reverse", false, "")
		flHeader      = flag.Bool("headers", false, "")
		flTLS         = flag.Bool("tls", false, "")
		flViewDNSInfo = flag.Bool("viewdns", false, "")
		flSRV         = flag.Bool("srv", false, "")
		flBing        = flag.String("bing", "", "")
		flYandex      = flag.String("yandex", "", "")
		flDomain      = flag.String("domain", "", "")
		flDictFile    = flag.String("dictionary", "", "")
		flFcrdns      = flag.Bool("fcrdns", false, "")
		flClean       = flag.Bool("clean", false, "")
		flCsv         = flag.Bool("csv", false, "")
		flJSON        = flag.Bool("json", false, "")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	// Modify timeout to Milliseconds for function calls
	if *flTimeout != 600 {
		*flTimeout = *flTimeout * 1000
	}

	// Holds all IP addresses for testing.
	ipAddrList := []string{}

	// Used to hold a ip or CIDR range passed as fl.Arg(0).

	// Verify that some sort of work load was given in commands.
	if *flIPFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain == "" && *flSRV == true {
		log.Fatal("SRV lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" && *flSRV == false {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Build list of domains.
	domains := []string{}
	if *flDomain != "" {
		if _, err := os.Stat(*flDomain); os.IsNotExist(err) {
			domains = append(domains, *flDomain)
		} else {
			lines, err := readFileLines(*flDomain)
			if err != nil {
				log.Fatal("Error reading " + *flDomain + " " + err.Error())
			}
			domains = append(domains, lines...)
		}
	}

	// Get first argument that is not an option and turn it into a list of IPs.
	if len(flag.Args()) > 0 {
		flNetwork := flag.Arg(0)
		list, err := linesToIPList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible IP or network into
	// a list of IPs. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid IP or CIDR range.
	if *flIPFile != "" {
		lines, err := readFileLines(*flIPFile)
		if err != nil {
			log.Fatal("Error reading " + *flIPFile + " " + err.Error())
		}
		list, err := linesToIPList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	// Use a map that acts like a set to store only unique results.
	resMap := make(map[bsw.Result]bool)

	// Start up *flConcurrency amount of goroutines.
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				task, result, err := def()
				if *flDebug == false {
					if m := c % 2; m == 0 {
						c = 3
						os.Stderr.WriteString("\rWorking \\")
					} else {
						c = 2
						os.Stderr.WriteString("\rWorking /")
					}
				}
				if err != nil && *flDebug {
					log.Printf("%v: %v", task, err.Error())
				}
				if err == nil {
					if *flDebug == true && len(result) > 0 {
						log.Printf("%v: %v %v: task completed successfully\n", task, result[0].Hostname, result[0].IP)
					}
					res <- result
				}
			}
			tracker <- empty{}
		}()
	}

	// Ingest incoming results.
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					} else {
						cfqdn, err := bsw.LookupCname(r.Hostname, *flServerAddr)
						if err == nil && len(cfqdn) > 0 {
							ip, err = bsw.LookupName(cfqdn, *flServerAddr)
							if err == nil && len(ip) > 0 {
								resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
							}
						}
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					}
				}
			} else {
				for _, r := range result {
					resMap[r] = true
				}
			}
		}
		tracker <- empty{}
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// IP based functionality should be added to the pool here.
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (string, bsw.Results, error) { return bsw.Reverse(host, *flServerAddr) }
		}
		if *flTLS {
			tasks <- func() (string, bsw.Results, error) { return bsw.TLS(host, *flTimeout) }
		}
		if *flViewDNSInfo {
			tasks <- func() (string, bsw.Results, error) { return bsw.ViewDNSInfo(host) }
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (string, bsw.Results, error) { return bsw.BingAPI(host, *flBing, bingPath) }
		}
		if *flHeader {
			tasks <- func() (string, bsw.Results, error) { return bsw.Headers(host, *flTimeout) }
		}
	}

	// Domain based functions will likely require separate blocks and should be added below.

	// Subdomain dictionary guessing.
	for _, d := range domains {
		domain := d
		if *flDictFile != "" {
			nameList, err := readFileLines(*flDictFile)
			if err != nil {
				log.Fatal("Error reading " + *flDictFile + " " + err.Error())
			}
			// Get an IP for a possible wildcard domain and use it as a blacklist.
			blacklist := bsw.GetWildCard(domain, *flServerAddr)
			var blacklist6 string
			if *flipv6 {
				blacklist6 = bsw.GetWildCard6(domain, *flServerAddr)
			}
			for _, n := range nameList {
				sub := n
				tasks <- func() (string, bsw.Results, error) { return bsw.Dictionary(domain, sub, blacklist, *flServerAddr) }
				if *flipv6 {
					tasks <- func() (string, bsw.Results, error) { return bsw.Dictionary6(domain, sub, blacklist6, *flServerAddr) }
				}
			}
		}

		if *flSRV != false {
			tasks <- func() (string, bsw.Results, error) { return bsw.SRV(domain, *flServerAddr) }
		}

		if *flYandex != "" {
			tasks <- func() (string, bsw.Results, error) { return bsw.YandexAPI(domain, *flYandex, *flServerAddr) }
		}
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer.
	<-tracker
	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")

	// Create a results slice from the unique set in resMap. Allows for sorting.
	results := bsw.Results{}
	for k := range resMap {
		results = append(results, k)
	}
	sort.Sort(results)

	switch {
	case *flJSON:
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	case *flCsv:
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	case *flClean:
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.IP] = append(cleanSet[r.IP], r.Hostname)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, h := range v {
				fmt.Printf("\t%s\n", h)
			}
		}
	default:
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"fmt"
	"sync"

	"github.com/jacobsa/comeback/kv"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
)

// Return a blob store that stores blobs in the supplied GCS bucket. GCS object
// names look like:
//
//     <prefix><score>
//
// where <score> is the result of calling Score.Hex.
//
// The blob store trusts that it has full ownership of this portion of the
// bucket's key space -- if a score name exists, then it points to the correct
// data.
func NewGCSStore(
	bucket gcs.Bucket,
	prefix string) (store Store)

type kvBasedBlobStore struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	kvStore kv.Store

	/////////////////////////
	// Constant data
	/////////////////////////

	keyPrefix           string
	maxBytesBuffered    int
	maxRequestsInFlight int

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// If any background write to the KV store has failed, an error that should
	// be returned for all future calls to Store or Flush.
	//
	// GUARDED_BY(mu)
	writeErr error

	// A map from scores that have been accepted by Store but not yet
	// successfully written out to the length of the corresponding blobs.
	//
	// INVARIANT: len(inFlight) <= maxRequestsInFlight
	//
	// GUARDED_BY(mu)
	inFlight map[Score]int

	// A cached sum of the lengths of in-flight scores.
	//
	// INVARIANT: bytesBuffered is the sum of all values of inFlight
	// INVARIANT: 0 <= bytesBuffered <= maxBytesBuffered
	//
	// GUARDED_BY(mu)
	bytesBuffered int

	// Signalled when the contents of inFlight change.
	inFlightChanged sync.Cond
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (s *kvBasedBlobStore) checkInvariants() {
	// INVARIANT: len(inFlight) <= maxRequestsInFlight
	if !(len(s.inFlight) <= s.maxRequestsInFlight) {
		panic(fmt.Sprintf("Too many in flight: %v", len(s.inFlight)))
	}

	// INVARIANT: bytesBuffered is the sum of all values of inFlight
	sum := 0
	for _, v := range s.inFlight {
		sum += v
	}

	if sum != s.bytesBuffered {
		panic(fmt.Sprintf("Differing sum: %v vs. %v", sum, s.bytesBuffered))
	}

	// INVARIANT: 0 <= bytesBuffered <= maxBytesBuffered
	if !(0 <= s.bytesBuffered && s.bytesBuffered <= s.maxBytesBuffered) {
		panic(fmt.Sprintf(
			"bytesBuffered of %v not in range [0, %v]",
			s.bytesBuffered,
			s.maxBytesBuffered))
	}
}

func (s *kvBasedBlobStore) makeKey(score Score) (key string) {
	key = s.keyPrefix + score.Hex()
	return
}

// LOCKS_REQURED(s.mu)
func (s *kvBasedBlobStore) hasRoom(blobLen int) bool {
	bytesLeft := s.maxBytesBuffered - s.bytesBuffered
	return len(s.inFlight) < s.maxRequestsInFlight && blobLen <= bytesLeft
}

// LOCKS_EXCLUDED(s.mu)
func (s *kvBasedBlobStore) makeDurable(score Score, key string, blob []byte) {
	var err error

	// When we exit, set a write error (if appropriate) and update the in-flight
	// map.
	defer func() {
		s.mu.Lock()
		defer s.mu.Unlock()

		// Is this the first write error?
		if err != nil && s.writeErr == nil {
			s.writeErr = err
		}

		delete(s.inFlight, score)
		s.bytesBuffered -= len(blob)
		s.inFlightChanged.Broadcast()
	}()

	// Attempt to write out the blob.
	err = s.kvStore.Set(key, blob)
	if err != nil {
		err = fmt.Errorf("kvStore.Set: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (s *kvBasedBlobStore) Store(blob []byte) (score Score, err error) {
	// Will this blob ever fit?
	if len(blob) > s.maxBytesBuffered {
		err = fmt.Errorf(
			"%v-byte blob is larger than buffer size of %v",
			len(blob),
			s.maxBytesBuffered)

		return
	}

	// Compute a score and a key for use under the lock below.
	score = ComputeScore(blob)
	key := s.makeKey(score)

	s.mu.Lock()
	defer s.mu.Unlock()

	// Wait until there is room for this request.
	for !s.hasRoom(len(blob)) {
		s.inFlightChanged.Wait()
	}

	// Have we already decided on a final error?
	if s.writeErr != nil {
		err = fmt.Errorf("Error from previous write: %v", s.writeErr)
		return
	}

	// If we already have a request for this score in flight, we can avoid
	// spawning another one. (If the existing one fails, the user will see an
	// error from Flush.)
	if _, ok := s.inFlight[score]; ok {
		return
	}

	// If the KV store already contains the key, we need not do anything further.
	if s.kvStore.Contains(key) {
		return
	}

	// Mark this blob as in flight.
	s.inFlight[score] = len(blob)
	s.bytesBuffered += len(blob)
	s.inFlightChanged.Broadcast()

	// Spawn a goroutine that will make the blob durable in the background.
	go s.makeDurable(score, key, blob)

	return
}

func (s *kvBasedBlobStore) Flush() (err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// Wait until there are no requests in flight.
	for len(s.inFlight) != 0 {
		s.inFlightChanged.Wait()
	}

	err = s.writeErr
	return
}

func (s *kvBasedBlobStore) Contains(score Score) (b bool) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// Do we have an in-flight request?
	if _, ok := s.inFlight[score]; ok {
		b = true
		return
	}

	// Does the key exist in the KV store?
	key := s.makeKey(score)
	if s.kvStore.Contains(key) {
		b = true
		return
	}

	return
}

func (s *kvBasedBlobStore) Load(score Score) (blob []byte, err error) {
	// Choose the appropriate key.
	key := s.makeKey(score)

	// Call the key/value store.
	if blob, err = s.kvStore.Get(key); err != nil {
		err = fmt.Errorf("Get: %v", err)
		return
	}

	return
}
<commit_msg>Trimmed junk.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"fmt"

	"github.com/jacobsa/gcloud/gcs"
)

// Return a blob store that stores blobs in the supplied GCS bucket. GCS object
// names look like:
//
//     <prefix><score>
//
// where <score> is the result of calling Score.Hex.
//
// The blob store trusts that it has full ownership of this portion of the
// bucket's key space -- if a score name exists, then it points to the correct
// data.
func NewGCSStore(
	bucket gcs.Bucket,
	prefix string) (store Store)

type gcsStore struct {
	bucket    gcs.Bucket
	keyPrefix string
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (s *kvBasedBlobStore) makeKey(score Score) (key string) {
	key = s.keyPrefix + score.Hex()
	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (s *kvBasedBlobStore) Store(blob []byte) (score Score, err error) {
	// Will this blob ever fit?
	if len(blob) > s.maxBytesBuffered {
		err = fmt.Errorf(
			"%v-byte blob is larger than buffer size of %v",
			len(blob),
			s.maxBytesBuffered)

		return
	}

	// Compute a score and a key for use under the lock below.
	score = ComputeScore(blob)
	key := s.makeKey(score)

	s.mu.Lock()
	defer s.mu.Unlock()

	// Wait until there is room for this request.
	for !s.hasRoom(len(blob)) {
		s.inFlightChanged.Wait()
	}

	// Have we already decided on a final error?
	if s.writeErr != nil {
		err = fmt.Errorf("Error from previous write: %v", s.writeErr)
		return
	}

	// If we already have a request for this score in flight, we can avoid
	// spawning another one. (If the existing one fails, the user will see an
	// error from Flush.)
	if _, ok := s.inFlight[score]; ok {
		return
	}

	// If the KV store already contains the key, we need not do anything further.
	if s.kvStore.Contains(key) {
		return
	}

	// Mark this blob as in flight.
	s.inFlight[score] = len(blob)
	s.bytesBuffered += len(blob)
	s.inFlightChanged.Broadcast()

	// Spawn a goroutine that will make the blob durable in the background.
	go s.makeDurable(score, key, blob)

	return
}

func (s *kvBasedBlobStore) Flush() (err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// Wait until there are no requests in flight.
	for len(s.inFlight) != 0 {
		s.inFlightChanged.Wait()
	}

	err = s.writeErr
	return
}

func (s *kvBasedBlobStore) Contains(score Score) (b bool) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// Do we have an in-flight request?
	if _, ok := s.inFlight[score]; ok {
		b = true
		return
	}

	// Does the key exist in the KV store?
	key := s.makeKey(score)
	if s.kvStore.Contains(key) {
		b = true
		return
	}

	return
}

func (s *kvBasedBlobStore) Load(score Score) (blob []byte, err error) {
	// Choose the appropriate key.
	key := s.makeKey(score)

	// Call the key/value store.
	if blob, err = s.kvStore.Get(key); err != nil {
		err = fmt.Errorf("Get: %v", err)
		return
	}

	return
}
<|endoftext|>


<commit_before>package herd

import (
	filegenclient "github.com/Symantec/Dominator/lib/filegen/client"
	"github.com/Symantec/Dominator/lib/mdb"
	"time"
)

func (herd *Herd) mdbUpdate(mdb *mdb.Mdb) {
	numNew, numDeleted, numChanged, wantedImages := herd.mdbUpdateGetLock(mdb)
	// Clean up unreferenced images.
	herd.imageManager.SetImageInterestList(wantedImages, true)
	pluralNew := "s"
	if numNew == 1 {
		pluralNew = ""
	}
	pluralDeleted := "s"
	if numDeleted == 1 {
		pluralDeleted = ""
	}
	pluralChanged := "s"
	if numChanged == 1 {
		pluralChanged = ""
	}
	herd.logger.Printf(
		"MDB update: %d new sub%s, %d removed sub%s, %d changed sub%s",
		numNew, pluralNew, numDeleted, pluralDeleted, numChanged, pluralChanged)
}

func (herd *Herd) mdbUpdateGetLock(mdb *mdb.Mdb) (
	int, int, int, map[string]struct{}) {
	herd.Lock()
	defer herd.Unlock()
	startTime := time.Now()
	numNew := 0
	numDeleted := 0
	numChanged := 0
	herd.subsByIndex = make([]*Sub, 0, len(mdb.Machines))
	// Mark for delete all current subs, then later unmark ones in the new MDB.
	subsToDelete := make(map[string]struct{})
	for _, sub := range herd.subsByName {
		subsToDelete[sub.mdb.Hostname] = struct{}{}
	}
	wantedImages := make(map[string]struct{})
	wantedImages[herd.defaultImageName] = struct{}{}
	for _, machine := range mdb.Machines { // Sorted by Hostname.
		sub := herd.subsByName[machine.Hostname]
		wantedImages[machine.RequiredImage] = struct{}{}
		wantedImages[machine.PlannedImage] = struct{}{}
		img := herd.imageManager.GetNoError(machine.RequiredImage)
		if sub == nil {
			sub = new(Sub)
			sub.herd = herd
			sub.mdb = machine
			herd.subsByName[machine.Hostname] = sub
			sub.fileUpdateChannel = herd.computedFilesManager.Add(
				filegenclient.Machine{machine, sub.getComputedFiles(img)}, 16)
			numNew++
		} else {
			if sub.mdb.RequiredImage != machine.RequiredImage {
				sub.computedInodes = nil
			}
			if sub.mdb != machine {
				sub.mdb = machine
				sub.generationCount = 0 // Force a full poll.
				herd.computedFilesManager.Update(
					filegenclient.Machine{machine, sub.getComputedFiles(img)})
				numChanged++
			}
		}
		delete(subsToDelete, machine.Hostname)
		herd.subsByIndex = append(herd.subsByIndex, sub)
		img = herd.imageManager.GetNoError(machine.PlannedImage)
		if img == nil {
			sub.havePlannedImage = false
		} else {
			sub.havePlannedImage = true
		}
	}
	delete(wantedImages, "")
	// Delete flagged subs (those not in the new MDB).
	for subHostname := range subsToDelete {
		herd.computedFilesManager.Remove(subHostname)
		sub := herd.subsByName[subHostname]
		sub.busyMutex.Lock()
		if sub.clientResource != nil {
			sub.clientResource.ScheduleClose()
		}
		sub.busyMutex.Unlock()
		delete(herd.subsByName, subHostname)
		numDeleted++
	}
	mdbUpdateTimeDistribution.Add(time.Since(startTime))
	return numNew, numDeleted, numChanged, wantedImages
}
<commit_msg>Use struct initialiser in dom/herd.Hard.mdbUpdateGetLock().<commit_after>package herd

import (
	filegenclient "github.com/Symantec/Dominator/lib/filegen/client"
	"github.com/Symantec/Dominator/lib/mdb"
	"time"
)

func (herd *Herd) mdbUpdate(mdb *mdb.Mdb) {
	numNew, numDeleted, numChanged, wantedImages := herd.mdbUpdateGetLock(mdb)
	// Clean up unreferenced images.
	herd.imageManager.SetImageInterestList(wantedImages, true)
	pluralNew := "s"
	if numNew == 1 {
		pluralNew = ""
	}
	pluralDeleted := "s"
	if numDeleted == 1 {
		pluralDeleted = ""
	}
	pluralChanged := "s"
	if numChanged == 1 {
		pluralChanged = ""
	}
	herd.logger.Printf(
		"MDB update: %d new sub%s, %d removed sub%s, %d changed sub%s",
		numNew, pluralNew, numDeleted, pluralDeleted, numChanged, pluralChanged)
}

func (herd *Herd) mdbUpdateGetLock(mdb *mdb.Mdb) (
	int, int, int, map[string]struct{}) {
	herd.Lock()
	defer herd.Unlock()
	startTime := time.Now()
	numNew := 0
	numDeleted := 0
	numChanged := 0
	herd.subsByIndex = make([]*Sub, 0, len(mdb.Machines))
	// Mark for delete all current subs, then later unmark ones in the new MDB.
	subsToDelete := make(map[string]struct{})
	for _, sub := range herd.subsByName {
		subsToDelete[sub.mdb.Hostname] = struct{}{}
	}
	wantedImages := make(map[string]struct{})
	wantedImages[herd.defaultImageName] = struct{}{}
	for _, machine := range mdb.Machines { // Sorted by Hostname.
		sub := herd.subsByName[machine.Hostname]
		wantedImages[machine.RequiredImage] = struct{}{}
		wantedImages[machine.PlannedImage] = struct{}{}
		img := herd.imageManager.GetNoError(machine.RequiredImage)
		if sub == nil {
			sub = &Sub{herd: herd, mdb: machine}
			herd.subsByName[machine.Hostname] = sub
			sub.fileUpdateChannel = herd.computedFilesManager.Add(
				filegenclient.Machine{machine, sub.getComputedFiles(img)}, 16)
			numNew++
		} else {
			if sub.mdb.RequiredImage != machine.RequiredImage {
				sub.computedInodes = nil
			}
			if sub.mdb != machine {
				sub.mdb = machine
				sub.generationCount = 0 // Force a full poll.
				herd.computedFilesManager.Update(
					filegenclient.Machine{machine, sub.getComputedFiles(img)})
				numChanged++
			}
		}
		delete(subsToDelete, machine.Hostname)
		herd.subsByIndex = append(herd.subsByIndex, sub)
		img = herd.imageManager.GetNoError(machine.PlannedImage)
		if img == nil {
			sub.havePlannedImage = false
		} else {
			sub.havePlannedImage = true
		}
	}
	delete(wantedImages, "")
	// Delete flagged subs (those not in the new MDB).
	for subHostname := range subsToDelete {
		herd.computedFilesManager.Remove(subHostname)
		sub := herd.subsByName[subHostname]
		sub.busyMutex.Lock()
		if sub.clientResource != nil {
			sub.clientResource.ScheduleClose()
		}
		sub.busyMutex.Unlock()
		delete(herd.subsByName, subHostname)
		numDeleted++
	}
	mdbUpdateTimeDistribution.Add(time.Since(startTime))
	return numNew, numDeleted, numChanged, wantedImages
}
<|endoftext|>


<commit_before>package driver

import (
	"database/sql"
	"database/sql/driver"
	"strconv"
	"time"
)

// Generic patterns in Google reports.
const (
	// Google can prefix a value by `auto:` or just return auto to symbolize an automatic strategy.
	auto      = "auto"
	autoValue = auto + ": "

	// Google uses `Excluded` to tag null value by context.
	excluded = "Excluded"

	// Google uses ` --` instead of an empty string to symbolize the fact that the field was never set.
	doubleDash = " --"

	// Google sometimes uses special value like `> 90%` or `< 10%`.
	almost10 = "< 10"
	almost90 = "> 90"
)

// PercentNullFloat64 represents a float64 that may be a percentage.
type PercentNullFloat64 struct {
	NullFloat64     sql.NullFloat64
	Almost, Percent bool
}

// Value implements the driver Valuer interface.
func (n PercentNullFloat64) Value() (driver.Value, error) {
	if !n.NullFloat64.Valid {
		return doubleDash, nil
	}
	var v string
	if n.Almost {
		if n.NullFloat64.Float64 > 90 {
			v = almost90
		} else {
			v = almost10
		}
	} else {
		v = strconv.FormatFloat(n.NullFloat64.Float64, 'f', 2, 64)
	}
	if n.Percent {
		return v + "%", nil
	}
	return v, nil
}

// AutoExcludedNullInt64 represents a int64 that may be null or defined as auto valuer.
type AutoExcludedNullInt64 struct {
	NullInt64 sql.NullInt64
	Auto,
	Excluded bool
}

// Value implements the driver Valuer interface.
func (n AutoExcludedNullInt64) Value() (driver.Value, error) {
	var v string
	if n.Auto {
		if !n.NullInt64.Valid {
			return auto, nil
		}
		v = autoValue
	}
	if n.Excluded {
		return excluded, nil
	}
	if !n.NullInt64.Valid {
		return doubleDash, nil
	}
	v += strconv.FormatInt(n.NullInt64.Int64, 10)

	return v, nil
}

// Float64 represents a float64 that may be rounded by using its precision.
type Float64 struct {
	Float64   float64
	Precision int
}

// Value implements the driver Valuer interface.
func (n Float64) Value() (driver.Value, error) {
	return strconv.FormatFloat(n.Float64, 'f', n.Precision, 64), nil
}

// NullString represents a string that may be null.
type NullString struct {
	String string
	Valid  bool // Valid is true if String is not NULL
}

// Value implements the driver Valuer interface.
func (n NullString) Value() (driver.Value, error) {
	if !n.Valid {
		return doubleDash, nil
	}
	return n.String, nil
}

// Time represents a Time that may be not set.
type Time struct {
	Time   time.Time
	Layout string
}

// Value implements the driver Valuer interface.
func (n Time) Value() (driver.Value, error) {
	if n.Time.IsZero() {
		return doubleDash, nil
	}
	return n.Time.Format(n.Layout), nil
}
<commit_msg>misspelling<commit_after>package driver

import (
	"database/sql"
	"database/sql/driver"
	"strconv"
	"time"
)

// Generic patterns in Google reports.
const (
	// Google can prefix a value by `auto:` or just return `auto` to symbolize an automatic strategy.
	auto      = "auto"
	autoValue = auto + ": "

	// Google uses `Excluded` to tag null value by context.
	excluded = "Excluded"

	// Google uses ` --` instead of an empty string to symbolize the fact that the field was never set.
	doubleDash = " --"

	// Google sometimes uses special value like `> 90%` or `< 10%`.
	almost10 = "< 10"
	almost90 = "> 90"
)

// PercentNullFloat64 represents a float64 that may be a percentage.
type PercentNullFloat64 struct {
	NullFloat64     sql.NullFloat64
	Almost, Percent bool
}

// Value implements the driver Valuer interface.
func (n PercentNullFloat64) Value() (driver.Value, error) {
	if !n.NullFloat64.Valid {
		return doubleDash, nil
	}
	var v string
	if n.Almost {
		if n.NullFloat64.Float64 > 90 {
			v = almost90
		} else {
			v = almost10
		}
	} else {
		v = strconv.FormatFloat(n.NullFloat64.Float64, 'f', 2, 64)
	}
	if n.Percent {
		return v + "%", nil
	}
	return v, nil
}

// AutoExcludedNullInt64 represents a int64 that may be null or defined as auto valuer.
type AutoExcludedNullInt64 struct {
	NullInt64 sql.NullInt64
	Auto,
	Excluded bool
}

// Value implements the driver Valuer interface.
func (n AutoExcludedNullInt64) Value() (driver.Value, error) {
	var v string
	if n.Auto {
		if !n.NullInt64.Valid {
			return auto, nil
		}
		v = autoValue
	}
	if n.Excluded {
		return excluded, nil
	}
	if !n.NullInt64.Valid {
		return doubleDash, nil
	}
	v += strconv.FormatInt(n.NullInt64.Int64, 10)

	return v, nil
}

// Float64 represents a float64 that may be rounded by using its precision.
type Float64 struct {
	Float64   float64
	Precision int
}

// Value implements the driver Valuer interface.
func (n Float64) Value() (driver.Value, error) {
	return strconv.FormatFloat(n.Float64, 'f', n.Precision, 64), nil
}

// NullString represents a string that may be null.
type NullString struct {
	String string
	Valid  bool // Valid is true if String is not NULL
}

// Value implements the driver Valuer interface.
func (n NullString) Value() (driver.Value, error) {
	if !n.Valid {
		return doubleDash, nil
	}
	return n.String, nil
}

// Time represents a Time that may be not set.
type Time struct {
	Time   time.Time
	Layout string
}

// Value implements the driver Valuer interface.
func (n Time) Value() (driver.Value, error) {
	if n.Time.IsZero() {
		return doubleDash, nil
	}
	return n.Time.Format(n.Layout), nil
}
<|endoftext|>


<commit_before>package main

const (
	sname           = "test.net.local"
	software        = "echochat"
	softwarev       = "v0.1"
	CONN_HOST       = "0.0.0.0"
	CONN_PORT       = "6667"
	ping_time       = 45 //something
	ping_check_time = 20 // time between the user's ping checks
	resolvehosts    = false
	isupport        = "NAMESX CHANTYPES=#& PREFIX=(o)@"
)

var (
	valid_chan_prefix = []string{"#", "&"}
	global_bad_chars  = []string{":", "!", "@", "*", "(", ")", "<", ">", ",", "~", "/", "\\"}
)
<commit_msg>Add comment to config.go<commit_after>package main

const (
	sname           = "test.net.local"
	software        = "echochat"
	softwarev       = "v0.1"
	CONN_HOST       = "0.0.0.0"
	CONN_PORT       = "6667"
	ping_time       = 45 //something
	ping_check_time = 20 // time between the user's ping checks
	resolvehosts    = false //Note: make forward confirmed reverse dns optional.
	isupport        = "NAMESX CHANTYPES=#& PREFIX=(o)@"
)

var (
	valid_chan_prefix = []string{"#", "&"}
	global_bad_chars  = []string{":", "!", "@", "*", "(", ")", "<", ">", ",", "~", "/", "\\"}
)
<|endoftext|>