<commit_before>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gtimer

import (
	"github.com/gogf/gf/container/garray"
	"github.com/gogf/gf/test/gtest"
	"testing"
	"time"
)

func TestTimer_Proceed(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		array := garray.New(true)
		timer := New(TimerOptions{
			Interval: time.Hour,
		})
		timer.Add(10000*time.Hour, func() {
			array.Append(1)
		})
		timer.proceed(10001)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 1)
		timer.proceed(20001)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 2)
	})
	gtest.C(t, func(t *gtest.T) {
		array := garray.New(true)
		timer := New(TimerOptions{
			Interval: time.Millisecond * 100,
		})
		timer.Add(10000*time.Hour, func() {
			array.Append(1)
		})
		ticks := int64((10000 * time.Hour) / (time.Millisecond * 100))
		timer.proceed(ticks + 1)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 1)
		timer.proceed(2*ticks + 1)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 2)
	})
}

func TestTimer_PriorityQueue(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		queue := newPriorityQueue()
		queue.Push(1, 1)
		queue.Push(4, 4)
		queue.Push(5, 5)
		queue.Push(2, 2)
		queue.Push(3, 3)
		t.Assert(queue.Pop(), 1)
		t.Assert(queue.Pop(), 2)
		t.Assert(queue.Pop(), 3)
		t.Assert(queue.Pop(), 4)
		t.Assert(queue.Pop(), 5)
	})
}
<commit_msg>add more unit testing case for package gtimer<commit_after>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gtimer

import (
	"github.com/gogf/gf/container/garray"
	"github.com/gogf/gf/test/gtest"
	"testing"
	"time"
)

func TestTimer_Proceed(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		array := garray.New(true)
		timer := New(TimerOptions{
			Interval: time.Hour,
		})
		timer.Add(10000*time.Hour, func() {
			array.Append(1)
		})
		timer.proceed(10001)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 1)
		timer.proceed(20001)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 2)
	})
	gtest.C(t, func(t *gtest.T) {
		array := garray.New(true)
		timer := New(TimerOptions{
			Interval: time.Millisecond * 100,
		})
		timer.Add(10000*time.Hour, func() {
			array.Append(1)
		})
		ticks := int64((10000 * time.Hour) / (time.Millisecond * 100))
		timer.proceed(ticks + 1)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 1)
		timer.proceed(2*ticks + 1)
		time.Sleep(10 * time.Millisecond)
		t.Assert(array.Len(), 2)
	})
}

func TestTimer_PriorityQueue(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		queue := newPriorityQueue()
		queue.Push(1, 1)
		queue.Push(4, 4)
		queue.Push(5, 5)
		queue.Push(2, 2)
		queue.Push(3, 3)
		t.Assert(queue.Pop(), 1)
		t.Assert(queue.Pop(), 2)
		t.Assert(queue.Pop(), 3)
		t.Assert(queue.Pop(), 4)
		t.Assert(queue.Pop(), 5)
	})
}

func TestTimer_PriorityQueue_FirstOneInArrayIsTheLeast(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		var (
			size  = 1000000
			array = garray.NewIntArrayRange(0, size, 1)
		)
		array.Shuffle()
		queue := newPriorityQueue()
		array.Iterator(func(k int, v int) bool {
			queue.Push(v, int64(v))
			return true
		})
		for i := 0; i < size; i++ {
			t.Assert(queue.Pop(), i)
			t.Assert(queue.heap.array[0].priority, i+1)
		}
	})
}
<|endoftext|>


<commit_before>package testutil

import (
	"os/exec"
	"strings"
	"testing"
	"time"

	zkutil "github.com/gigawattio/zklib/util"

	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
)

var (
	zkTimeout       = 5 * time.Second
	zkDeleteRetries = 10
)

// withZk allows for a default ZooKeeper address to be specified, and if the
// port is reachable that connection info will be used.  Otherwise a full test
// cluster is launched (NB: this can take 6+ seconds).
func WithZk(t *testing.T, size int, defaultServer string, fn func(zkServers []string)) {
	if size == 1 && len(defaultServer) > 0 {
		checkCmd := exec.Command("nc", append([]string{"-w", "1"}, strings.Split(defaultServer, ":")...)...)
		if err := checkCmd.Run(); err == nil {
			log.Infof("Using already-running default ZooKeeper@%v", defaultServer)
			fn([]string{defaultServer})
			return
		}
	}
	log.Infof("Starting a new ZooKeeper test cluster..")
	WithTestZkCluster(t, size, func(zkServers []string) {
		time.Sleep(100 * time.Millisecond) // Give ZooKeeper a moment to start up.
		fn(zkServers)
	})
}

func ResetZk(t *testing.T, zkServers []string, path string) {
	conn, zkEvents, err := zk.Connect(zkServers, zkTimeout)
	if err != nil {
		t.Fatal(err)
	}
	WhenZkHasSession(zkEvents, func() {
		if err := zkutil.RecursivelyDelete(conn, path, zkDeleteRetries); err != nil {
			t.Fatal(err)
		}
	})
}
<commit_msg>Doc fix.<commit_after>package testutil

import (
	"os/exec"
	"strings"
	"testing"
	"time"

	zkutil "github.com/gigawattio/zklib/util"

	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
)

var (
	zkTimeout       = 5 * time.Second
	zkDeleteRetries = 10
)

// WithZk allows for a default ZooKeeper address to be specified, and if the
// port is reachable that connection info will be used.  Otherwise a full test
// cluster is launched (NB: this can take 6+ seconds).
func WithZk(t *testing.T, size int, defaultServer string, fn func(zkServers []string)) {
	if size == 1 && len(defaultServer) > 0 {
		checkCmd := exec.Command("nc", append([]string{"-w", "1"}, strings.Split(defaultServer, ":")...)...)
		if err := checkCmd.Run(); err == nil {
			log.Infof("Using already-running default ZooKeeper@%v", defaultServer)
			fn([]string{defaultServer})
			return
		}
	}
	log.Infof("Starting a new ZooKeeper test cluster..")
	WithTestZkCluster(t, size, func(zkServers []string) {
		time.Sleep(100 * time.Millisecond) // Give ZooKeeper a moment to start up.
		fn(zkServers)
	})
}

func ResetZk(t *testing.T, zkServers []string, path string) {
	conn, zkEvents, err := zk.Connect(zkServers, zkTimeout)
	if err != nil {
		t.Fatal(err)
	}
	WhenZkHasSession(zkEvents, func() {
		if err := zkutil.RecursivelyDelete(conn, path, zkDeleteRetries); err != nil {
			t.Fatal(err)
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package timing

import (
	"errors"
	"sync"
	"sync/atomic"
	"time"

	l "github.com/soumya92/barista/logging"
	"github.com/soumya92/barista/notifier"
)

// scheduler implements a Scheduler tied to actual time.
type scheduler struct {
	sync.Mutex

	timer  *time.Timer
	ticker *time.Ticker

	notifyFn func()
	notifyCh <-chan struct{}

	waiting int32 // basically bool, but we need atomics.
}

var (
	// A set of channels to be closed by timing.Resume.
	// This allows schedulers to wait for resume, without
	// requiring a reference to each created scheduler.
	waiters  []chan struct{}
	paused   = false
	testMode = false

	mu sync.Mutex
)

// NewScheduler creates a new scheduler.
func NewScheduler() Scheduler {
	fn, ch := notifier.New()
	s := &scheduler{notifyFn: fn, notifyCh: ch}
	l.Attach(s, ch, "")
	if testMode {
		return &testScheduler{scheduler: s}
	}
	return s
}

// Pause timing.
func Pause() {
	mu.Lock()
	defer mu.Unlock()
	paused = true
}

// Await waits for the bar to resume timing.
// It returns immediately if the bar is not paused.
func Await() {
	mu.Lock()
	if !paused {
		mu.Unlock()
		return
	}
	ch := make(chan struct{})
	waiters = append(waiters, ch)
	mu.Unlock()
	<-ch
}

// Resume timing.
func Resume() {
	mu.Lock()
	defer mu.Unlock()
	paused = false
	for _, ch := range waiters {
		close(ch)
	}
	waiters = nil
}

// Tick returns a channel that receives an empty value
// when the scheduler is triggered.
func (s *scheduler) Tick() <-chan struct{} {
	return s.notifyCh
}

func (s *scheduler) At(when time.Time) Scheduler {
	l.Fine("%s At(%v)", l.ID(s), when)
	s.Lock()
	defer s.Unlock()
	s.stop()
	s.timer = time.AfterFunc(when.Sub(Now()), s.maybeTrigger)
	return s
}

func (s *scheduler) After(delay time.Duration) Scheduler {
	l.Fine("%s After(%v)", l.ID(s), delay)
	s.Lock()
	defer s.Unlock()
	s.stop()
	s.timer = time.AfterFunc(delay, s.maybeTrigger)
	return s
}

func (s *scheduler) Every(interval time.Duration) Scheduler {
	l.Fine("%s Every(%v)", l.ID(s), interval)
	if interval <= 0 {
		panic(errors.New("non-positive interval for Scheduler#Every"))
	}
	s.Lock()
	defer s.Unlock()
	s.stop()
	s.ticker = time.NewTicker(interval)
	go func() {
		s.Lock()
		ticker := s.ticker
		s.Unlock()
		if ticker == nil {
			// Scheduler stopped before goroutine was started.
			return
		}
		for range ticker.C {
			go s.maybeTrigger()
		}
	}()
	return s
}

func (s *scheduler) Stop() {
	l.Fine("%s Stop", l.ID(s))
	s.Lock()
	defer s.Unlock()
	s.stop()
}

func (s *scheduler) maybeTrigger() {
	if !atomic.CompareAndSwapInt32(&s.waiting, 0, 1) {
		return
	}
	Await()
	if atomic.CompareAndSwapInt32(&s.waiting, 1, 0) {
		s.notifyFn()
	}
}

func (s *scheduler) stop() {
	if s.timer != nil {
		s.timer.Stop()
		s.timer = nil
	}
	if s.ticker != nil {
		s.ticker.Stop()
		s.ticker = nil
	}
}
<commit_msg>timing.await is not yet a public API<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package timing

import (
	"errors"
	"sync"
	"sync/atomic"
	"time"

	l "github.com/soumya92/barista/logging"
	"github.com/soumya92/barista/notifier"
)

// scheduler implements a Scheduler tied to actual time.
type scheduler struct {
	sync.Mutex

	timer  *time.Timer
	ticker *time.Ticker

	notifyFn func()
	notifyCh <-chan struct{}

	waiting int32 // basically bool, but we need atomics.
}

var (
	// A set of channels to be closed by timing.Resume.
	// This allows schedulers to wait for resume, without
	// requiring a reference to each created scheduler.
	waiters  []chan struct{}
	paused   = false
	testMode = false

	mu sync.Mutex
)

// NewScheduler creates a new scheduler.
func NewScheduler() Scheduler {
	fn, ch := notifier.New()
	s := &scheduler{notifyFn: fn, notifyCh: ch}
	l.Attach(s, ch, "")
	if testMode {
		return &testScheduler{scheduler: s}
	}
	return s
}

// Pause timing.
func Pause() {
	mu.Lock()
	defer mu.Unlock()
	paused = true
}

// await waits for the bar to resume timing.
// It returns immediately if the bar is not paused.
func await() {
	mu.Lock()
	if !paused {
		mu.Unlock()
		return
	}
	ch := make(chan struct{})
	waiters = append(waiters, ch)
	mu.Unlock()
	<-ch
}

// Resume timing.
func Resume() {
	mu.Lock()
	defer mu.Unlock()
	paused = false
	for _, ch := range waiters {
		close(ch)
	}
	waiters = nil
}

// Tick returns a channel that receives an empty value
// when the scheduler is triggered.
func (s *scheduler) Tick() <-chan struct{} {
	return s.notifyCh
}

func (s *scheduler) At(when time.Time) Scheduler {
	l.Fine("%s At(%v)", l.ID(s), when)
	s.Lock()
	defer s.Unlock()
	s.stop()
	s.timer = time.AfterFunc(when.Sub(Now()), s.maybeTrigger)
	return s
}

func (s *scheduler) After(delay time.Duration) Scheduler {
	l.Fine("%s After(%v)", l.ID(s), delay)
	s.Lock()
	defer s.Unlock()
	s.stop()
	s.timer = time.AfterFunc(delay, s.maybeTrigger)
	return s
}

func (s *scheduler) Every(interval time.Duration) Scheduler {
	l.Fine("%s Every(%v)", l.ID(s), interval)
	if interval <= 0 {
		panic(errors.New("non-positive interval for Scheduler#Every"))
	}
	s.Lock()
	defer s.Unlock()
	s.stop()
	s.ticker = time.NewTicker(interval)
	go func() {
		s.Lock()
		ticker := s.ticker
		s.Unlock()
		if ticker == nil {
			// Scheduler stopped before goroutine was started.
			return
		}
		for range ticker.C {
			go s.maybeTrigger()
		}
	}()
	return s
}

func (s *scheduler) Stop() {
	l.Fine("%s Stop", l.ID(s))
	s.Lock()
	defer s.Unlock()
	s.stop()
}

func (s *scheduler) maybeTrigger() {
	if !atomic.CompareAndSwapInt32(&s.waiting, 0, 1) {
		return
	}
	await()
	if atomic.CompareAndSwapInt32(&s.waiting, 1, 0) {
		s.notifyFn()
	}
}

func (s *scheduler) stop() {
	if s.timer != nil {
		s.timer.Stop()
		s.timer = nil
	}
	if s.ticker != nil {
		s.ticker.Stop()
		s.ticker = nil
	}
}
<|endoftext|>


<commit_before>/**
 * Local request handlers that perform higher level logic.
 */
package local

import "bufio"
import "bytes"
import "encoding/binary"
import "fmt"
import "io"
import "math"

import "../binprot"
import "../common"

// Chunk size, leaving room for the token
// Make sure the value subtracted from chunk size stays in sync
// with the size of the Metadata struct
const CHUNK_SIZE = 1024 - 16
const FULL_DATA_SIZE = 1024

func consumeResponseHeader(localReader *bufio.Reader) error {
    resHeader, err := binprot.ReadResponseHeader(localReader)
    if err != nil {
        return err
    }

    err = binprot.DecodeError(resHeader)
    if err != nil {
        return err
    }

    return nil
}

func HandleSet(cmd common.SetRequest, remoteReader *bufio.Reader, localReader *bufio.Reader, localWriter *bufio.Writer) error {
    // Read in the data from the remote connection
    buf := make([]byte, cmd.Length)
    _, err := io.ReadFull(remoteReader, buf)
    // TODO: text protocol bytes?

    numChunks := int(math.Ceil(float64(cmd.Length) / float64(CHUNK_SIZE)))
    token := <-tokens

    metaKey := metaKey(cmd.Key)
    metaData := common.Metadata {
        Length:    cmd.Length,
        OrigFlags: cmd.Flags,
        NumChunks: uint32(numChunks),
        ChunkSize: CHUNK_SIZE,
        Token:     *token,
    }

    metaDataBuf := new(bytes.Buffer)
    binary.Write(metaDataBuf, binary.BigEndian, metaData)

    // Write metadata key
    // TODO: should there be a unique flags value for chunked data?
    localCmd := binprot.SetCmd(metaKey, cmd.Flags, cmd.Exptime, common.METADATA_SIZE)
    err = setLocal(localWriter, localCmd, nil, metaDataBuf.Bytes())
    if err != nil {
        return err
    }

    // Read server's response
    err = consumeResponseHeader(localReader)
    if err != nil {
        return err
    }

    // Write all the data chunks
    // TODO: Clean up if a data chunk write fails
    // Failure can mean the write failing at the I/O level
    // or at the memcached level, e.g. response == ERROR
    for i := 0; i < numChunks; i++ {
        // Build this chunk's key
        key := chunkKey(cmd.Key, i)

        // indices for slicing, end exclusive
        start, end := chunkSliceIndices(CHUNK_SIZE, i, int(cmd.Length))
        chunkBuf := buf[start:end]

        // Pad the data to always be CHUNK_SIZE
        if (end-start) < CHUNK_SIZE {
            padding := CHUNK_SIZE - (end-start)
            padtext := bytes.Repeat([]byte{byte(0)}, padding)
            chunkBuf = append(chunkBuf, padtext...)
        }

        // Write the key
        localCmd = binprot.SetCmd(key, cmd.Flags, cmd.Exptime, FULL_DATA_SIZE)
        err = setLocal(localWriter, localCmd, token, chunkBuf)
        if err != nil {
            return err
        }

        // Read server's response
        err := consumeResponseHeader(localReader)
        if err != nil {
            return err
        }
    }

    return nil
}

func HandleGet(cmd common.GetRequest, localReader *bufio.Reader, localWriter *bufio.Writer) (chan common.GetResponse, chan error) {
    // No buffering here so there's not multiple gets in memory
    dataOut := make(chan common.GetResponse)
    errorOut := make(chan error)
    go realHandleGet(cmd, dataOut, errorOut, localReader, localWriter)
    return dataOut, errorOut
}

func realHandleGet(cmd common.GetRequest, dataOut chan common.GetResponse, errorOut chan error,
                   localReader *bufio.Reader, localWriter *bufio.Writer) {
    // read index
    // make buf
    // for numChunks do
    //   read chunk, append to buffer
    // send response

    defer close(errorOut)
    defer close(dataOut)

    outer: for _, key := range cmd.Keys {
        _, metaData, err := getMetadata(localReader, localWriter, key)
        if err != nil {
            if err == common.MISS || err == common.ERROR_KEY_NOT_FOUND {
                fmt.Println("Get miss because of missing metadata. Key:", key)
                dataOut <- common.GetResponse {
                    Miss: true,
                    Key:  key,
                    //opaque
                }
                continue outer
            }

            errorOut <- err
            return
        }

        // Retrieve all the data from memcached
        dataBuf := make([]byte, metaData.Length)
        tokenBuf := make([]byte, 16)

        for i := 0; i < int(metaData.NumChunks); i++ {
            chunkKey := chunkKey(key, i)

            // indices for slicing, end exclusive
            start, end := chunkSliceIndices(int(metaData.ChunkSize), i, int(metaData.Length))

            // Get the data directly into our buf
            chunkBuf := dataBuf[start:end]
            getCmd := binprot.GetCmd(chunkKey)
            err = getLocalIntoBuf(localReader, localWriter, getCmd, tokenBuf, chunkBuf, int(metaData.ChunkSize))

            if err != nil {
                if err == common.MISS || err == common.ERROR_KEY_NOT_FOUND {
                    fmt.Println("Get miss because of missing chunk. Cmd:", getCmd)
                    dataOut <- common.GetResponse {
                        Miss: true,
                        Key:  key,
                        //opaque
                    }
                    continue outer
                }

                errorOut <- err
                return
            }

            if (!bytes.Equal(metaData.Token[:], tokenBuf)) {
                fmt.Println("Get miss because of invalid chunk token. Cmd:", getCmd)
                fmt.Printf("Expected: %v\n", metaData.Token)
                fmt.Printf("Got:      %v\n", tokenBuf)
                dataOut <- common.GetResponse {
                    Miss: true,
                    Key:  key,
                    //opaque
                }
                continue outer
            }
        }

        dataOut <- common.GetResponse {
            Miss:     false,
            Key:      key,
            // opaque
            Metadata: metaData,
            Data:     dataBuf,
        }
    }
}

func HandleDelete(cmd common.DeleteRequest, localReader *bufio.Reader, localWriter *bufio.Writer) error {
    // read metadata
    // delete metadata
    // for 0 to metadata.numChunks
    //  delete item

    metaKey, metaData, err := getMetadata(localReader, localWriter, cmd.Key)

    if err != nil {
        if err == common.MISS {
            fmt.Println("Delete miss because of missing metadata. Key:", cmd.Key)
        }
        return err
    }

    deleteCmd := binprot.DeleteCmd(metaKey)
    err = simpleCmdLocal(localReader, localWriter, deleteCmd)
    if err != nil {
        return err
    }

    for i := 0; i < int(metaData.NumChunks); i++ {
        chunkKey := chunkKey(cmd.Key, i)
        deleteCmd = binprot.DeleteCmd(chunkKey)
        err := simpleCmdLocal(localReader, localWriter, deleteCmd)
        if err != nil {
            return err
        }
    }

    return nil
}

func HandleTouch(cmd common.TouchRequest, localReader *bufio.Reader, localWriter *bufio.Writer) error {
    // read metadata
    // for 0 to metadata.numChunks
    //  touch item
    // touch metadata

    metaKey, metaData, err := getMetadata(localReader, localWriter, cmd.Key)

    if err != nil {
        if err == common.MISS {
            fmt.Println("Touch miss because of missing metadata. Key:", cmd.Key)
            return err
        }

        return err
    }

    for i := 0; i < int(metaData.NumChunks); i++ {
        chunkKey := chunkKey(cmd.Key, i)
        touchCmd := binprot.TouchCmd(chunkKey, cmd.Exptime)
        err := simpleCmdLocal(localReader, localWriter, touchCmd)
        if err != nil {
            return err
        }
    }

    touchCmd := binprot.TouchCmd(metaKey, cmd.Exptime)
    err = simpleCmdLocal(localReader, localWriter, touchCmd)
    if err != nil {
        return err
    }

    return nil
}
<commit_msg>reminders<commit_after>/**
 * Local request handlers that perform higher level logic.
 */
package local

import "bufio"
import "bytes"
import "encoding/binary"
import "fmt"
import "io"
import "math"

import "../binprot"
import "../common"

// Chunk size, leaving room for the token
// Make sure the value subtracted from chunk size stays in sync
// with the size of the Metadata struct
const CHUNK_SIZE = 1024 - 16
const FULL_DATA_SIZE = 1024

func consumeResponseHeader(localReader *bufio.Reader) error {
    resHeader, err := binprot.ReadResponseHeader(localReader)
    if err != nil {
        return err
    }

    err = binprot.DecodeError(resHeader)
    if err != nil {
        return err
    }

    return nil
}

func HandleSet(cmd common.SetRequest, remoteReader *bufio.Reader, localReader *bufio.Reader, localWriter *bufio.Writer) error {
    // Read in the data from the remote connection
    buf := make([]byte, cmd.Length)
    _, err := io.ReadFull(remoteReader, buf)
    // TODO: text protocol bytes?

    numChunks := int(math.Ceil(float64(cmd.Length) / float64(CHUNK_SIZE)))
    token := <-tokens

    metaKey := metaKey(cmd.Key)
    metaData := common.Metadata {
        Length:    cmd.Length,
        OrigFlags: cmd.Flags,
        NumChunks: uint32(numChunks),
        ChunkSize: CHUNK_SIZE,
        Token:     *token,
    }

    metaDataBuf := new(bytes.Buffer)
    binary.Write(metaDataBuf, binary.BigEndian, metaData)

    // Write metadata key
    // TODO: should there be a unique flags value for chunked data?
    localCmd := binprot.SetCmd(metaKey, cmd.Flags, cmd.Exptime, common.METADATA_SIZE)
    err = setLocal(localWriter, localCmd, nil, metaDataBuf.Bytes())
    if err != nil {
        return err
    }

    // Read server's response
    err = consumeResponseHeader(localReader)
    if err != nil {
        return err
    }

    // Write all the data chunks
    // TODO: Clean up if a data chunk write fails
    // Failure can mean the write failing at the I/O level
    // or at the memcached level, e.g. response == ERROR
    for i := 0; i < numChunks; i++ {
        // Build this chunk's key
        key := chunkKey(cmd.Key, i)

        // indices for slicing, end exclusive
        start, end := chunkSliceIndices(CHUNK_SIZE, i, int(cmd.Length))
        chunkBuf := buf[start:end]

        // Pad the data to always be CHUNK_SIZE
        if (end-start) < CHUNK_SIZE {
            padding := CHUNK_SIZE - (end-start)
            padtext := bytes.Repeat([]byte{byte(0)}, padding)
            chunkBuf = append(chunkBuf, padtext...)
        }

        // Write the key
        localCmd = binprot.SetCmd(key, cmd.Flags, cmd.Exptime, FULL_DATA_SIZE)
        err = setLocal(localWriter, localCmd, token, chunkBuf)
        if err != nil {
            return err
        }

        // Read server's response
        err := consumeResponseHeader(localReader)
        if err != nil {
            return err
        }
    }

    return nil
}

func HandleGet(cmd common.GetRequest, localReader *bufio.Reader, localWriter *bufio.Writer) (chan common.GetResponse, chan error) {
    // No buffering here so there's not multiple gets in memory
    dataOut := make(chan common.GetResponse)
    errorOut := make(chan error)
    go realHandleGet(cmd, dataOut, errorOut, localReader, localWriter)
    return dataOut, errorOut
}

func realHandleGet(cmd common.GetRequest, dataOut chan common.GetResponse, errorOut chan error,
                   localReader *bufio.Reader, localWriter *bufio.Writer) {
    // read index
    // make buf
    // for numChunks do
    //   read chunk, append to buffer
    // send response

    defer close(errorOut)
    defer close(dataOut)

    outer: for _, key := range cmd.Keys {
        _, metaData, err := getMetadata(localReader, localWriter, key)
        if err != nil {
            // TODO: Better error management
            if err == common.MISS || err == common.ERROR_KEY_NOT_FOUND {
                fmt.Println("Get miss because of missing metadata. Key:", key)
                dataOut <- common.GetResponse {
                    Miss: true,
                    Key:  key,
                    //opaque
                }
                continue outer
            }

            errorOut <- err
            return
        }

        // Retrieve all the data from memcached
        dataBuf := make([]byte, metaData.Length)
        tokenBuf := make([]byte, 16)

        for i := 0; i < int(metaData.NumChunks); i++ {
            chunkKey := chunkKey(key, i)

            // indices for slicing, end exclusive
            start, end := chunkSliceIndices(int(metaData.ChunkSize), i, int(metaData.Length))

            // Get the data directly into our buf
            chunkBuf := dataBuf[start:end]
            getCmd := binprot.GetCmd(chunkKey)
            err = getLocalIntoBuf(localReader, localWriter, getCmd, tokenBuf, chunkBuf, int(metaData.ChunkSize))

            if err != nil {
                // TODO: Better error management
                if err == common.MISS || err == common.ERROR_KEY_NOT_FOUND {
                    fmt.Println("Get miss because of missing chunk. Cmd:", getCmd)
                    dataOut <- common.GetResponse {
                        Miss: true,
                        Key:  key,
                        //opaque
                    }
                    continue outer
                }

                errorOut <- err
                return
            }

            if (!bytes.Equal(metaData.Token[:], tokenBuf)) {
                fmt.Println("Get miss because of invalid chunk token. Cmd:", getCmd)
                fmt.Printf("Expected: %v\n", metaData.Token)
                fmt.Printf("Got:      %v\n", tokenBuf)
                dataOut <- common.GetResponse {
                    Miss: true,
                    Key:  key,
                    //opaque
                }
                continue outer
            }
        }

        dataOut <- common.GetResponse {
            Miss:     false,
            Key:      key,
            // opaque
            Metadata: metaData,
            Data:     dataBuf,
        }
    }
}

func HandleDelete(cmd common.DeleteRequest, localReader *bufio.Reader, localWriter *bufio.Writer) error {
    // read metadata
    // delete metadata
    // for 0 to metadata.numChunks
    //  delete item

    metaKey, metaData, err := getMetadata(localReader, localWriter, cmd.Key)

    if err != nil {
        if err == common.MISS {
            fmt.Println("Delete miss because of missing metadata. Key:", cmd.Key)
        }
        return err
    }

    deleteCmd := binprot.DeleteCmd(metaKey)
    err = simpleCmdLocal(localReader, localWriter, deleteCmd)
    if err != nil {
        return err
    }

    for i := 0; i < int(metaData.NumChunks); i++ {
        chunkKey := chunkKey(cmd.Key, i)
        deleteCmd = binprot.DeleteCmd(chunkKey)
        err := simpleCmdLocal(localReader, localWriter, deleteCmd)
        if err != nil {
            return err
        }
    }

    return nil
}

func HandleTouch(cmd common.TouchRequest, localReader *bufio.Reader, localWriter *bufio.Writer) error {
    // read metadata
    // for 0 to metadata.numChunks
    //  touch item
    // touch metadata

    metaKey, metaData, err := getMetadata(localReader, localWriter, cmd.Key)

    if err != nil {
        if err == common.MISS {
            fmt.Println("Touch miss because of missing metadata. Key:", cmd.Key)
            return err
        }

        return err
    }

    for i := 0; i < int(metaData.NumChunks); i++ {
        chunkKey := chunkKey(cmd.Key, i)
        touchCmd := binprot.TouchCmd(chunkKey, cmd.Exptime)
        err := simpleCmdLocal(localReader, localWriter, touchCmd)
        if err != nil {
            return err
        }
    }

    touchCmd := binprot.TouchCmd(metaKey, cmd.Exptime)
    err = simpleCmdLocal(localReader, localWriter, touchCmd)
    if err != nil {
        return err
    }

    return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"flag"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"io"
	"net"
	"log"
	"os"
	"strings"
	"time"
	"strconv"

	"github.com/tarm/serial"
	"gopkg.in/readline.v1"
)

var (
	rlInstance *readline.Instance
	conn       io.ReadWriter
	serIn      = make(chan []byte)
	outBound   = make(chan string)
	progress   = make(chan bool, 1)
	incLevel   = make(chan int)

	port    = flag.String("p", "", "serial port (required: /dev/tty* or COM*)")
	baud    = flag.Int("b", 115200, "baud rate")
	upload  = flag.String("u", "", "upload the specified firmware, then quit")
	expand  = flag.String("e", "", "expand specified file to stdout, then quit")
	verbose = flag.Bool("v", false, "verbose output, for debugging only")
	capture = flag.String("c", "", "a file where captured output is appended")
	timeout = flag.Duration("t", 500*time.Millisecond, "serial echo timeout")
)

func main() {
	flag.Parse()
	var err error

	// expansion does not use the serial port, it just expands include lines
	if *expand != "" {
		expandFile()
		return
	}

	if *port == "" || flag.NArg() > 0 {
		flag.PrintDefaults()
		os.Exit(1)
	}
	conn, err = connect(*port)
	check(err)
	//defer conn.Close()
	fmt.Println("Connected to:", *port)

	go serialInput() // feed the serIn channel

	// firmware upload uses serial in a different way, needs to quit when done
	if *upload != "" {
		firmwareUpload()
		return
	}

	rlInstance, err = readline.NewEx(&readline.Config{
		UniqueEditLine: true,
	})
	check(err)
	defer rlInstance.Close()

	go serialExchange()

	//conn.Flush()
	readWithTimeout() // get rid of partial pending data
	for {
		line, err := rlInstance.Readline()
		if err != nil { // io.EOF, readline.ErrInterrupt
			break
		}
		parseAndSend(line)
		if strings.HasPrefix(line, "include ") {
			fmt.Println("\\ done.")
		}
	}
}

// isNetPort returns true if the argument is of the form "...:<N>"
func isNetPort(s string) bool {
	if n := strings.Index(s, ":"); n > 0 {
		p, e := strconv.Atoi(s[n+1:])
		return e == nil && p > 0
	}
	return false
}

// connect to either a net port for telnet use, or to a serial device
func connect(name string) (io.ReadWriter, error) {
	if isNetPort(name) {
		return net.Dial("tcp", name)
	}

	config := serial.Config{Name: *port, Baud: *baud}
	if *upload != "" {
		config.Parity = serial.ParityEven
	}
	return serial.OpenPort(&config)
}

func parseAndSend(line string) {
	if strings.HasPrefix(line, "include ") {
		for _, fname := range strings.Split(line[8:], " ") {
			doInclude(fname)
		}
	} else {
		outBound <- line
		<-progress
	}
}

func check(err error) {
	if err != nil {
		if rlInstance != nil {
			rlInstance.Close()
		}
		log.Fatal(err)
	}
}

func serialInput() {
	var f *os.File
	if *capture != "" {
		var err error
		opts := os.O_WRONLY | os.O_APPEND | os.O_CREATE
		f, err = os.OpenFile(*capture, opts, 0666)
		check(err)
		defer f.Close()
	}
	for {
		buf := make([]byte, 600)
		n, err := conn.Read(buf)
		check(err)
		if n == 0 {
			close(serIn)
			return
		}
		if f != nil {
			f.Write(buf[:n])
		}
		serIn <- buf[:n]
	}
}

func readWithTimeout() []byte {
	select {
	case data := <-serIn:
		return data
	case <-time.After(*timeout):
		return nil
	}
}

func serialExchange() {
	depth := 0
	for {
		select {

		case data := <-serIn:
			if len(data) == 0 {
				return
			}
			fmt.Print(string(data))

		case line := <-outBound:
			immediate := depth == 0
			// the task here is to omit "normal" output for included lines,
			// i.e. lines which only generate an echo, a space, and " ok.\n"
			// everything else should be echoed in full, including the input
			if len(line) > 0 {
				serialSend(line)
				prefix, matched := expectEcho(line, false, func(s string) {
					fmt.Print(s) // called to flush pending serial input lines
				})
				fmt.Print(prefix)
				if matched && immediate {
					fmt.Print(line)
					line = ""
				}
			}
			// now that the echo is done, send a CR and wait for the prompt
			serialSend("\r")
			prompt := " ok.\n"
			prefix, matched := expectEcho(prompt, immediate, func(s string) {
				fmt.Print(line + s) // show original command first
				line = ""
			})
			if !matched {
				prompt = ""
			}
			if immediate || prefix != " " || !matched {
				fmt.Print(line + prefix + prompt)
			}
			// signal to sender that this request has been processed
			progress <- matched

		case n := <-incLevel:
			depth += n
		}
	}
}

func expectEcho(match string, immed bool, flusher func(string)) (string, bool) {
	var collected []byte
	for {
		data := readWithTimeout()
		collected = append(collected, data...)
		if bytes.HasSuffix(collected, []byte(match)) {
			bytesBefore := len(collected) - len(match)
			return string(collected[:bytesBefore]), true
		}
		if immed || len(data) == 0 {
			return string(collected), false
		}
		if n := bytes.LastIndexByte(collected, '\n'); n >= 0 {
			flusher(string(collected[:n+1]))
			collected = collected[n+1:]
		}
	}
}

func serialSend(data string) {
	_, err := conn.Write([]byte(data))
	check(err)
}

func doInclude(fname string) {
	if fname == "" {
		return // silently ignore empty files
	}

	incLevel <- +1
	defer func() { incLevel <- -1 }()

	lineNum := 0
	fmt.Printf("\\       >>> include %s\n", fname)
	defer func() {
		fmt.Printf("\\       <<<<<<<<<<< %s (%d lines)\n", fname, lineNum)
	}()

	f, err := os.Open(fname)
	if err != nil {
		fmt.Println(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		lineNum++

		s := strings.TrimLeft(line, " ")
		if s == "" || strings.HasPrefix(s, "\\ ") {
			if len(*expand) == 0 {
				continue // don't send empty or comment-only lines
			}
		}

		parseAndSend(line)
	}
}

func expandFile() {
	go func() {
		for line := range outBound {
			fmt.Println(line)
			progress <- true
		}
	}()

	go func() {
		for range incLevel {
		}
	}()

	for _, fname := range strings.Split(*expand, ",") {
		doInclude(fname)
	}
}

func hexToBin(data []byte) []byte {
	var bin []byte
	for _, line := range strings.Split(string(data), "\n") {
		if strings.HasSuffix(line, "\r") {
			line = line[:len(line)-1]
		}
		if len(line) == 0 {
			continue
		}
		if line[0] != ':' || len(line) < 11 {
			fmt.Println("Not ihex format:", line)
			os.Exit(1)
		}
		bytes, err := hex.DecodeString(line[1:])
		check(err)
		if bytes[3] != 0x00 {
			continue
		}
		offset := (int(bytes[1]) << 8) + int(bytes[2])
		length := bytes[0]
		for offset > len(bin) {
			bin = append(bin, 0xFF)
		}
		bin = append(bin, bytes[4:4+length]...)
	}
	return bin
}

func firmwareUpload() {
	f, err := os.Open(*upload)
	if err != nil {
		fmt.Println(err)
	}
	defer f.Close()

	data, err := ioutil.ReadAll(f)
	check(err)

	// convert to binary if first bytes look like they are in hex format
	tag := ""
	if len(data) > 11 && data[0] == ':' {
		_, err = hex.DecodeString(string(data[1:11]))
		if err == nil {
			data = hexToBin(data)
			tag = " (converted from Intel HEX)"
		}
	}

	fmt.Printf("        File: %s\n", *upload)
	fmt.Printf("       Count: %d bytes%s\n", len(data), tag)
	fmt.Printf("    Checksum: %08x hex\n", crc32.ChecksumIEEE(data))

	uploadSTM32(data)
}
<commit_msg>cleanup<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"flag"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/tarm/serial"
	"gopkg.in/readline.v1"
)

var (
	rlInstance *readline.Instance
	conn       io.ReadWriter
	serIn      = make(chan []byte)
	outBound   = make(chan string)
	progress   = make(chan bool, 1)
	incLevel   = make(chan int)

	port    = flag.String("p", "", "serial port (required: /dev/tty* or COM*)")
	baud    = flag.Int("b", 115200, "baud rate")
	upload  = flag.String("u", "", "upload the specified firmware, then quit")
	expand  = flag.String("e", "", "expand specified file to stdout, then quit")
	verbose = flag.Bool("v", false, "verbose output, for debugging only")
	capture = flag.String("c", "", "a file where captured output is appended")
	timeout = flag.Duration("t", 500*time.Millisecond, "serial echo timeout")
)

func main() {
	flag.Parse()
	var err error

	// expansion does not use the serial port, it just expands include lines
	if *expand != "" {
		expandFile()
		return
	}

	if *port == "" || flag.NArg() > 0 {
		flag.PrintDefaults()
		os.Exit(1)
	}
	conn, err = connect(*port)
	check(err)
	//defer conn.Close()
	fmt.Println("Connected to:", *port)

	go serialInput() // feed the serIn channel

	// firmware upload uses serial in a different way, needs to quit when done
	if *upload != "" {
		firmwareUpload()
		return
	}

	rlInstance, err = readline.NewEx(&readline.Config{
		UniqueEditLine: true,
	})
	check(err)
	defer rlInstance.Close()

	go serialExchange()

	//conn.Flush()
	readWithTimeout() // get rid of partial pending data
	for {
		line, err := rlInstance.Readline()
		if err != nil { // io.EOF, readline.ErrInterrupt
			break
		}
		parseAndSend(line)
		if strings.HasPrefix(line, "include ") {
			fmt.Println("\\ done.")
		}
	}
}

// isNetPort returns true if the argument is of the form "...:<N>"
func isNetPort(s string) bool {
	if n := strings.Index(s, ":"); n > 0 {
		p, e := strconv.Atoi(s[n+1:])
		return e == nil && p > 0
	}
	return false
}

// connect to either a net port for telnet use, or to a serial device
func connect(name string) (io.ReadWriter, error) {
	if isNetPort(name) {
		return net.Dial("tcp", name)
	}

	config := serial.Config{Name: *port, Baud: *baud}
	if *upload != "" {
		config.Parity = serial.ParityEven
	}
	return serial.OpenPort(&config)
}

func parseAndSend(line string) {
	if strings.HasPrefix(line, "include ") {
		for _, fname := range strings.Split(line[8:], " ") {
			doInclude(fname)
		}
	} else {
		outBound <- line
		<-progress
	}
}

func check(err error) {
	if err != nil {
		if rlInstance != nil {
			rlInstance.Close()
		}
		log.Fatal(err)
	}
}

func serialInput() {
	var f *os.File
	if *capture != "" {
		var err error
		opts := os.O_WRONLY | os.O_APPEND | os.O_CREATE
		f, err = os.OpenFile(*capture, opts, 0666)
		check(err)
		defer f.Close()
	}
	for {
		buf := make([]byte, 600)
		n, err := conn.Read(buf)
		check(err)
		if n == 0 {
			close(serIn)
			return
		}
		if f != nil {
			f.Write(buf[:n])
		}
		serIn <- buf[:n]
	}
}

func readWithTimeout() []byte {
	select {
	case data := <-serIn:
		return data
	case <-time.After(*timeout):
		return nil
	}
}

func serialExchange() {
	depth := 0
	for {
		select {

		case data := <-serIn:
			if len(data) == 0 {
				return
			}
			fmt.Print(string(data))

		case line := <-outBound:
			immediate := depth == 0
			// the task here is to omit "normal" output for included lines,
			// i.e. lines which only generate an echo, a space, and " ok.\n"
			// everything else should be echoed in full, including the input
			if len(line) > 0 {
				serialSend(line)
				prefix, matched := expectEcho(line, false, func(s string) {
					fmt.Print(s) // called to flush pending serial input lines
				})
				fmt.Print(prefix)
				if matched && immediate {
					fmt.Print(line)
					line = ""
				}
			}
			// now that the echo is done, send a CR and wait for the prompt
			serialSend("\r")
			prompt := " ok.\n"
			prefix, matched := expectEcho(prompt, immediate, func(s string) {
				fmt.Print(line + s) // show original command first
				line = ""
			})
			if !matched {
				prompt = ""
			}
			if immediate || prefix != " " || !matched {
				fmt.Print(line + prefix + prompt)
			}
			// signal to sender that this request has been processed
			progress <- matched

		case n := <-incLevel:
			depth += n
		}
	}
}

func expectEcho(match string, immed bool, flusher func(string)) (string, bool) {
	var collected []byte
	for {
		data := readWithTimeout()
		collected = append(collected, data...)
		if bytes.HasSuffix(collected, []byte(match)) {
			bytesBefore := len(collected) - len(match)
			return string(collected[:bytesBefore]), true
		}
		if immed || len(data) == 0 {
			return string(collected), false
		}
		if n := bytes.LastIndexByte(collected, '\n'); n >= 0 {
			flusher(string(collected[:n+1]))
			collected = collected[n+1:]
		}
	}
}

func serialSend(data string) {
	_, err := conn.Write([]byte(data))
	check(err)
}

func doInclude(fname string) {
	if fname == "" {
		return // silently ignore empty files
	}

	incLevel <- +1
	defer func() { incLevel <- -1 }()

	lineNum := 0
	fmt.Printf("\\       >>> include %s\n", fname)
	defer func() {
		fmt.Printf("\\       <<<<<<<<<<< %s (%d lines)\n", fname, lineNum)
	}()

	f, err := os.Open(fname)
	if err != nil {
		fmt.Println(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		lineNum++

		s := strings.TrimLeft(line, " ")
		if s == "" || strings.HasPrefix(s, "\\ ") {
			if len(*expand) == 0 {
				continue // don't send empty or comment-only lines
			}
		}

		parseAndSend(line)
	}
}

func expandFile() {
	go func() {
		for line := range outBound {
			fmt.Println(line)
			progress <- true
		}
	}()

	go func() {
		for range incLevel {
		}
	}()

	for _, fname := range strings.Split(*expand, ",") {
		doInclude(fname)
	}
}

func hexToBin(data []byte) []byte {
	var bin []byte
	for _, line := range strings.Split(string(data), "\n") {
		if strings.HasSuffix(line, "\r") {
			line = line[:len(line)-1]
		}
		if len(line) == 0 {
			continue
		}
		if line[0] != ':' || len(line) < 11 {
			fmt.Println("Not ihex format:", line)
			os.Exit(1)
		}
		bytes, err := hex.DecodeString(line[1:])
		check(err)
		if bytes[3] != 0x00 {
			continue
		}
		offset := (int(bytes[1]) << 8) + int(bytes[2])
		length := bytes[0]
		for offset > len(bin) {
			bin = append(bin, 0xFF)
		}
		bin = append(bin, bytes[4:4+length]...)
	}
	return bin
}

func firmwareUpload() {
	f, err := os.Open(*upload)
	if err != nil {
		fmt.Println(err)
	}
	defer f.Close()

	data, err := ioutil.ReadAll(f)
	check(err)

	// convert to binary if first bytes look like they are in hex format
	tag := ""
	if len(data) > 11 && data[0] == ':' {
		_, err = hex.DecodeString(string(data[1:11]))
		if err == nil {
			data = hexToBin(data)
			tag = " (converted from Intel HEX)"
		}
	}

	fmt.Printf("        File: %s\n", *upload)
	fmt.Printf("       Count: %d bytes%s\n", len(data), tag)
	fmt.Printf("    Checksum: %08x hex\n", crc32.ChecksumIEEE(data))

	uploadSTM32(data)
}
<|endoftext|>


<commit_before>package atc

import (
	"fmt"
	"path/filepath"
	"strings"
)

type PathFlag string

func (path *PathFlag) UnmarshalFlag(value string) error {
	if value == "" {
		return nil
	}

	matches, err := filepath.Glob(value)
	if err != nil {
		return fmt.Errorf("failed to expand path '%s': %s", value, err)
	}

	if len(matches) == 0 {
		return fmt.Errorf("path '%s' does not exist", value)
	}

	if len(matches) > 1 {
		return fmt.Errorf("path '%s' resolves to multiple entries: %s", value, strings.Join(matches, ", "))
	}

	*path = PathFlag(matches[0])
	return nil
}
<commit_msg>Add tab completion for fly paths.<commit_after>package atc

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/jessevdk/go-flags"
)

type PathFlag string

func (path *PathFlag) UnmarshalFlag(value string) error {
	if value == "" {
		return nil
	}

	matches, err := filepath.Glob(value)
	if err != nil {
		return fmt.Errorf("failed to expand path '%s': %s", value, err)
	}

	if len(matches) == 0 {
		return fmt.Errorf("path '%s' does not exist", value)
	}

	if len(matches) > 1 {
		return fmt.Errorf("path '%s' resolves to multiple entries: %s", value, strings.Join(matches, ", "))
	}

	*path = PathFlag(matches[0])
	return nil
}

func (path *PathFlag) Complete(match string) []flags.Completion {
	matches, _ := filepath.Glob(match + "*")
	comps := make([]flags.Completion, len(matches))

	for i, v := range matches {
		comps[i].Item = v
	}

	return comps
}
<|endoftext|>


<commit_before>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package main

import (
	"fmt"
	"sync"

	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uiprogress/util/strutil"
	"github.com/satori/go.uuid"
	"github.com/topfreegames/khan/models"
	"github.com/topfreegames/khan/util"
)

func createTestData(db models.DB, games, clansPerGame, playersPerClan, playersWithoutClan, pendingMembershipsPerClan int, progress func() bool) error {

	wg := new(sync.WaitGroup)
	wg.Add(games)

	for gameIndex := 0; gameIndex < games; gameIndex++ {
		go func(progress func() bool, wg *sync.WaitGroup) {
			game := models.GameFactory.MustCreateWithOption(util.JSON{
				"PublicID": uuid.NewV4().String(),
			}).(*models.Game)
			err := db.Insert(game)
			if err != nil {
				panic(err.Error())
			}
			progress()

			for playerIndex := 0; playerIndex < playersWithoutClan; playerIndex++ {
				player := models.PlayerFactory.MustCreateWithOption(util.JSON{
					"GameID":   game.PublicID,
					"PublicID": uuid.NewV4().String(),
				}).(*models.Player)
				err = db.Insert(player)
				if err != nil {
					panic(err.Error())
				}
				progress()
			}

			for clanIndex := 0; clanIndex < clansPerGame; clanIndex++ {
				owner := models.PlayerFactory.MustCreateWithOption(util.JSON{
					"GameID":   game.PublicID,
					"PublicID": uuid.NewV4().String(),
				}).(*models.Player)
				err = db.Insert(owner)
				if err != nil {
					panic(err.Error())
				}

				clan := models.ClanFactory.MustCreateWithOption(util.JSON{
					"GameID":   game.PublicID,
					"PublicID": uuid.NewV4().String(),
					"OwnerID":  owner.ID,
				}).(*models.Clan)
				err = db.Insert(clan)
				if err != nil {
					panic(err.Error())
				}
				progress()

				for playerIndex := 0; playerIndex < playersPerClan; playerIndex++ {
					player := models.PlayerFactory.MustCreateWithOption(util.JSON{
						"GameID":   game.PublicID,
						"PublicID": uuid.NewV4().String(),
					}).(*models.Player)
					err = db.Insert(player)
					if err != nil {
						panic(err.Error())
					}

					membership := models.MembershipFactory.MustCreateWithOption(util.JSON{
						"GameID":      game.PublicID,
						"PlayerID":    player.ID,
						"ClanID":      clan.ID,
						"RequestorID": owner.ID,
						"Metadata":    util.JSON{"x": "a"},
						"Accepted":    true,
					}).(*models.Membership)

					err = db.Insert(membership)
					if err != nil {
						panic(err.Error())
					}
					progress()
				}

				for membershipIndex := 0; membershipIndex < pendingMembershipsPerClan; membershipIndex++ {
					player := models.PlayerFactory.MustCreateWithOption(util.JSON{
						"GameID":   game.PublicID,
						"PublicID": uuid.NewV4().String(),
					}).(*models.Player)
					err = db.Insert(player)
					if err != nil {
						panic(err.Error())
					}

					membership := models.MembershipFactory.MustCreateWithOption(util.JSON{
						"GameID":      game.PublicID,
						"PlayerID":    player.ID,
						"ClanID":      clan.ID,
						"RequestorID": owner.ID,
						"Metadata":    util.JSON{"x": "a"},
					}).(*models.Membership)

					err = db.Insert(membership)
					if err != nil {
						panic(err.Error())
					}
					progress()
				}
			}

			wg.Done()
		}(progress, wg)
	}

	wg.Wait()

	return nil
}

func main() {
	games := 20
	clansPerGame := 1000
	playersPerClan := 50
	playersWithoutClan := 500000
	pendingMembershipsPerClan := 250

	//games := 20
	//clansPerGame := 10
	//playersPerClan := 50
	//playersWithoutClan := 50
	//pendingMembershipsPerClan := 25

	totalClans := games * clansPerGame
	totalPlayers := totalClans*playersPerClan + games*playersWithoutClan
	totalPending := totalClans * pendingMembershipsPerClan
	totalOps := games + totalClans + totalPlayers + totalPending

	uiprogress.Start()                     // start rendering
	bar := uiprogress.AddBar(totalOps - 1) // Add a new bar
	bar.AppendCompleted()
	bar.PrependElapsed()

	// prepend the deploy step to the bar
	bar.PrependFunc(func(b *uiprogress.Bar) string {
		text := fmt.Sprintf("%d/%d", b.Current()+1, totalOps)
		return strutil.Resize(text, uint(len(text)))
	})

	testDb, err := models.GetPerfDB()
	if err != nil {
		panic(err.Error())
	}

	createTestData(testDb, games, clansPerGame, playersPerClan, playersWithoutClan, pendingMembershipsPerClan, bar.Incr)
}
<commit_msg>Generating perf db now tells us how long til finish<commit_after>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package main

import (
	"fmt"
	"time"

	"github.com/gosuri/uiprogress"
	"github.com/gosuri/uiprogress/util/strutil"
	"github.com/satori/go.uuid"
	"github.com/topfreegames/khan/models"
	"github.com/topfreegames/khan/util"
)

func createTestData(db models.DB, games, clansPerGame, playersPerClan, playersWithoutClan, pendingMembershipsPerClan int, progress func() bool) error {
	for gameIndex := 0; gameIndex < games; gameIndex++ {
		game := models.GameFactory.MustCreateWithOption(util.JSON{
			"PublicID": uuid.NewV4().String(),
		}).(*models.Game)
		err := db.Insert(game)
		if err != nil {
			panic(err.Error())
		}
		progress()

		for playerIndex := 0; playerIndex < playersWithoutClan; playerIndex++ {
			player := models.PlayerFactory.MustCreateWithOption(util.JSON{
				"GameID":   game.PublicID,
				"PublicID": uuid.NewV4().String(),
			}).(*models.Player)
			err = db.Insert(player)
			if err != nil {
				panic(err.Error())
			}
			progress()
		}

		for clanIndex := 0; clanIndex < clansPerGame; clanIndex++ {
			owner := models.PlayerFactory.MustCreateWithOption(util.JSON{
				"GameID":   game.PublicID,
				"PublicID": uuid.NewV4().String(),
			}).(*models.Player)
			err = db.Insert(owner)
			if err != nil {
				panic(err.Error())
			}

			clan := models.ClanFactory.MustCreateWithOption(util.JSON{
				"GameID":   game.PublicID,
				"PublicID": uuid.NewV4().String(),
				"OwnerID":  owner.ID,
			}).(*models.Clan)
			err = db.Insert(clan)
			if err != nil {
				panic(err.Error())
			}
			progress()

			for playerIndex := 0; playerIndex < playersPerClan; playerIndex++ {
				player := models.PlayerFactory.MustCreateWithOption(util.JSON{
					"GameID":   game.PublicID,
					"PublicID": uuid.NewV4().String(),
				}).(*models.Player)
				err = db.Insert(player)
				if err != nil {
					panic(err.Error())
				}

				membership := models.MembershipFactory.MustCreateWithOption(util.JSON{
					"GameID":      game.PublicID,
					"PlayerID":    player.ID,
					"ClanID":      clan.ID,
					"RequestorID": owner.ID,
					"Metadata":    util.JSON{"x": "a"},
					"Accepted":    true,
				}).(*models.Membership)

				err = db.Insert(membership)
				if err != nil {
					panic(err.Error())
				}
				progress()
			}

			for membershipIndex := 0; membershipIndex < pendingMembershipsPerClan; membershipIndex++ {
				player := models.PlayerFactory.MustCreateWithOption(util.JSON{
					"GameID":   game.PublicID,
					"PublicID": uuid.NewV4().String(),
				}).(*models.Player)
				err = db.Insert(player)
				if err != nil {
					panic(err.Error())
				}

				membership := models.MembershipFactory.MustCreateWithOption(util.JSON{
					"GameID":      game.PublicID,
					"PlayerID":    player.ID,
					"ClanID":      clan.ID,
					"RequestorID": owner.ID,
					"Metadata":    util.JSON{"x": "a"},
				}).(*models.Membership)

				err = db.Insert(membership)
				if err != nil {
					panic(err.Error())
				}
				progress()
			}
		}
	}

	return nil
}

func main() {
	start := time.Now()

	games := 20
	clansPerGame := 1000
	playersPerClan := 50
	playersWithoutClan := 500000
	pendingMembershipsPerClan := 250

	totalClans := games * clansPerGame
	totalPlayers := totalClans*playersPerClan + games*playersWithoutClan
	totalPending := totalClans * pendingMembershipsPerClan
	totalOps := games + totalClans + totalPlayers + totalPending

	uiprogress.Start()                     // start rendering
	bar := uiprogress.AddBar(totalOps - 1) // Add a new bar
	bar.AppendCompleted()
	bar.PrependElapsed()

	// prepend the deploy step to the bar
	bar.PrependFunc(func(b *uiprogress.Bar) string {
		ellapsed := time.Now().Sub(start)
		itemsPerSec := float64(b.Current()+1) / ellapsed.Seconds()
		timeToComplete := float64(totalOps) / itemsPerSec / 60.0 / 60.0
		text := fmt.Sprintf("%d/%d (%.2fhs to complete)", b.Current()+1, totalOps, timeToComplete)
		return strutil.Resize(text, uint(len(text)))
	})

	testDb, err := models.GetPerfDB()
	if err != nil {
		panic(err.Error())
	}

	createTestData(testDb, games, clansPerGame, playersPerClan, playersWithoutClan, pendingMembershipsPerClan, bar.Incr)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/couchbase/go-couchbase"
	"log"
	"runtime"
	"sync"
	"time"
)

func maybeFatal(err error) {
	if err != nil {
		log.Fatalf("Error:  %v", err)
	}
}

var serverURL = flag.String("serverURL", "http://localhost:9000",
	"couchbase server URL")
var poolName = flag.String("poolName", "default",
	"pool name")
var bucketName = flag.String("bucketName", "default",
	"bucket name")
var set = flag.Bool("set", false, "create document mode")
var size = flag.Int("size", 1024, "document size")
var documents = flag.Int("documents", 2000000, "total documents")
var threads = flag.Int("threads", 10, "Number of threads")
var quantum = flag.Int("quantum", 1024, "Number of documents per bulkGet")

var wg sync.WaitGroup

func main() {
	flag.Parse()

	runtime.GOMAXPROCS(*threads)
	client, err := couchbase.Connect(*serverURL)
	if err != nil {
		log.Printf("Connect failed %v", err)
		return
	}

	cbpool, err := client.GetPool("default")
	if err != nil {
		log.Printf("Failed to connect to default pool %v", err)
		return
	}

	var cbbucket *couchbase.Bucket
	cbbucket, err = cbpool.GetBucket(*bucketName)

	start := time.Now()
	if *set == true {
		value := generateRandomDoc(*size)
		for i := 0; i < *threads; i++ {
			go doSetOps(cbbucket, i, *documents / *threads, string(value))
			wg.Add(1)
		}
	} else {
		for i := 0; i < *threads; i++ {
			go doBulkGetOps(cbbucket, *documents / *threads, *quantum, i*(*documents / *threads))
			wg.Add(1)
		}
	}

	wg.Wait()

	finish := time.Now().Sub(start)
	fmt.Printf("**** Did %d ops in %s. Ops/sec %d\n",
		*documents, finish.String(), int(float64(*documents)/finish.Seconds()))

	if err != nil {
		log.Printf("Failed to connect to bucket %v", err)
		return
	}

}

func doBulkGetOps(b *couchbase.Bucket, total int, quantum int, startNum int) {

	defer wg.Done()
	start := time.Now()
	iter := total / quantum
	currentKeyNum := startNum
	for i := 0; i < iter; i++ {

		keylist := make([]string, quantum, quantum)
		for j := 0; j < quantum; j++ {
			key := fmt.Sprintf("test%d", currentKeyNum)
			keylist[j] = key
			currentKeyNum++

		}
		_, err := b.GetBulk(keylist)
		if err != nil {
			log.Printf(" Failed to get keys startnum %s to %d", keylist[0], quantum)
		}
	}
	fmt.Printf("Did %d ops in %s\n",
		total, time.Now().Sub(start).String())
}

func generateRandomDoc(size int) []byte {

	rb := make([]byte, size)
	_, err := rand.Read(rb)

	if err != nil {
		log.Fatal("Cannot generate data %v", err)
	}

	rs := base64.URLEncoding.EncodeToString(rb)
	data := map[string]interface{}{"data": rs}

	encode, _ := json.Marshal(data)
	return encode

}

func doSetOps(b *couchbase.Bucket, startNum int, total int, data string) {

	defer wg.Done()
	start := time.Now()
	for i := 0; i < total; i++ {
		k := fmt.Sprintf("test%d", startNum+i)
		maybeFatal(b.Set(k, 0, data))
	}
	fmt.Printf("Did %d ops in %s\n",
		total, time.Now().Sub(start).String())
}
<commit_msg>fix starting count in create document mode<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/couchbase/go-couchbase"
	"log"
	"runtime"
	"sync"
	"time"
)

func maybeFatal(err error) {
	if err != nil {
		log.Fatalf("Error:  %v", err)
	}
}

var serverURL = flag.String("serverURL", "http://localhost:9000",
	"couchbase server URL")
var poolName = flag.String("poolName", "default",
	"pool name")
var bucketName = flag.String("bucketName", "default",
	"bucket name")
var set = flag.Bool("set", false, "create document mode")
var size = flag.Int("size", 1024, "document size")
var documents = flag.Int("documents", 2000000, "total documents")
var threads = flag.Int("threads", 10, "Number of threads")
var quantum = flag.Int("quantum", 1024, "Number of documents per bulkGet")

var wg sync.WaitGroup

func main() {
	flag.Parse()

	runtime.GOMAXPROCS(*threads)
	client, err := couchbase.Connect(*serverURL)
	if err != nil {
		log.Printf("Connect failed %v", err)
		return
	}

	cbpool, err := client.GetPool("default")
	if err != nil {
		log.Printf("Failed to connect to default pool %v", err)
		return
	}

	var cbbucket *couchbase.Bucket
	cbbucket, err = cbpool.GetBucket(*bucketName)

	start := time.Now()
	if *set == true {
		value := generateRandomDoc(*size)
		for i := 0; i < *threads; i++ {
			go doSetOps(cbbucket, i*(*documents / *threads), *documents / *threads, string(value))
			wg.Add(1)
		}
	} else {
		for i := 0; i < *threads; i++ {
			go doBulkGetOps(cbbucket, *documents / *threads, *quantum, i*(*documents / *threads))
			wg.Add(1)
		}
	}

	wg.Wait()

	finish := time.Now().Sub(start)
	fmt.Printf("**** Did %d ops in %s. Ops/sec %d\n",
		*documents, finish.String(), int(float64(*documents)/finish.Seconds()))

	if err != nil {
		log.Printf("Failed to connect to bucket %v", err)
		return
	}

}

func doBulkGetOps(b *couchbase.Bucket, total int, quantum int, startNum int) {

	defer wg.Done()
	start := time.Now()
	iter := total / quantum
	currentKeyNum := startNum
	for i := 0; i < iter; i++ {

		keylist := make([]string, quantum, quantum)
		for j := 0; j < quantum; j++ {
			key := fmt.Sprintf("test%d", currentKeyNum)
			keylist[j] = key
			currentKeyNum++

		}
		_, err := b.GetBulk(keylist)
		if err != nil {
			log.Printf(" Failed to get keys startnum %s to %d", keylist[0], quantum)
		}
	}
	fmt.Printf("Did %d ops in %s\n",
		total, time.Now().Sub(start).String())
}

func generateRandomDoc(size int) []byte {

	rb := make([]byte, size)
	_, err := rand.Read(rb)

	if err != nil {
		log.Fatal("Cannot generate data %v", err)
	}

	rs := base64.URLEncoding.EncodeToString(rb)
	data := map[string]interface{}{"data": rs}

	encode, _ := json.Marshal(data)
	return encode

}

func doSetOps(b *couchbase.Bucket, startNum int, total int, data string) {

	defer wg.Done()
	start := time.Now()
	for i := 0; i < total; i++ {
		k := fmt.Sprintf("test%d", startNum+i)
		maybeFatal(b.Set(k, 0, data))
	}
	fmt.Printf("Did %d ops in %s\n",
		total, time.Now().Sub(start).String())
}
<|endoftext|>


<commit_before>package pggameday

import (
	"database/sql"
	"github.com/ecopony/gamedayapi"
	_ "github.com/lib/pq" // PostgreSQL driver
	"log"
	s "strings"
)

// CreateTables creates the database tables used by the importer. For now it's just the pitches table.
// This will drop tables if they already exist.
func CreateTables() {
	log.Println("Creating database tables.")

	// Assumes a pg database exists named go-gameday, a role that can access it.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	log.Println("\t-Creating pitches table")
	db.Exec("DROP INDEX IF EXISTS pitches_game_id")
	db.Exec("DROP TABLE IF EXISTS pitches")
	db.Exec(`CREATE TABLE pitches (pitchid SERIAL PRIMARY KEY, game_id varchar(40), year int, inning int, half varchar(6),
		at_bat_num int, at_bat_b int, at_bat_s int, at_bat_o int, at_bat_start_tfs int, batter int, stand char(1), b_height
		varchar(4), pitcher int, p_throws char(1), at_bat_des varchar(400), at_bat_event varchar(20), pitch_des varchar(40),
		pitch_id int, pitch_type char(1), pitch_pitch_type char(2), type_confidence DECIMAL(4, 3), pitch_tfs int,
	  	pitch_x DECIMAL(5, 2), pitch_y DECIMAL(5, 2), pitch_sv_id varchar(40), pitch_start_speed DECIMAL(4, 1),
	  	pitch_end_speed DECIMAL(4, 1), sz_top DECIMAL(3, 2), sz_bottom DECIMAL(3, 2), pfx_x DECIMAL(4, 2), pfx_z
	  	DECIMAL(4, 2), px DECIMAL(4, 3), pz DECIMAL(4, 3), x0 DECIMAL(5, 3), y0 DECIMAL(5, 3), z0 DECIMAL(5, 3), vx0
	  	DECIMAL(4, 2), vy0 DECIMAL(6, 3), vz0 DECIMAL(5, 3), ax DECIMAL(5, 3), ay DECIMAL(5, 3), az DECIMAL(5, 3), break_y
	  	DECIMAL(3, 1), break_angle DECIMAL(4, 1), break_length DECIMAL(3, 1), zone int, spin_dir DECIMAL(6, 3), spin_rate
	  	DECIMAL(7, 3), nasty int)`)
	log.Println("\t-Creating pitches index")
	db.Exec("CREATE INDEX pitches_game_id ON pitches (game_id)")
	db.Exec("CREATE UNIQUE INDEX pitches_game_id_pitch_id ON pitches (game_id, pitch_id)")

	log.Println("\t-Creating players table")
	db.Exec("DROP INDEX IF EXISTS players_id")
	db.Exec("DROP TABLE IF EXISTS players")
	db.Exec(`CREATE TABLE players (playerid SERIAL PRIMARY KEY, id int, first varchar(40), last varchar(40), num int,
		boxname varchar(40), rl varchar(1), bats varchar(1), position varchar(2), current_position varchar(2), status
		varchar(1), team_abbrev varchar(3), team_id int, parent_team_abbrev varchar(3), parent_team_id int, bat_order int,
		game_position varchar(2), avg DECIMAL(4, 3), hr int, rbi int, wins int, losses int, era DECIMAL(5, 2))`)
	log.Println("\t-Creating players index")
	db.Exec("CREATE UNIQUE INDEX players_id ON players (id)")

	log.Println("Done.")
}

// ImportPitchesForTeamAndYears saves all pitch data fields for a team and season.
func ImportPitchesForTeamAndYears(teamCode string, years []int) {
	log.Println("Importing pitches for " + teamCode)

	// Assumes a pg database exists named go-gameday, a role that can access it.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	issue := db.Ping()
	log.Println(issue)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	fetchFunction := func(game *gamedayapi.Game) {
		log.Println(">>>> " + game.ID + " <<<<")

		var pitchCount int
		err := db.QueryRow("SELECT count(*) FROM pitches WHERE game_id = $1", game.ID).Scan(&pitchCount)
		if err != nil {
			log.Fatal(err)
		}
		if pitchCount > 0 {
			return
		}

		for _, inning := range game.AllInnings().Innings {
			half := "top"
			for _, atBat := range inning.AtBats() {
				if atBat.O == "3" {
					half = "bottom"
				}
				for _, pitch := range atBat.Pitches {
					// log.Println("> " + pitch.SzTop)

					res, err := db.Query(`INSERT INTO pitches
						(game_id, year, inning, half, at_bat_num, at_bat_b, at_bat_s, at_bat_o, at_bat_start_tfs,
						batter, stand, b_height, pitcher, p_throws, at_bat_des, at_bat_event,
						pitch_des, pitch_id, pitch_type, pitch_pitch_type, type_confidence,
						pitch_tfs, pitch_x, pitch_y, pitch_sv_id, pitch_start_speed, pitch_end_speed,
						sz_top, sz_bottom, pfx_x, pfx_z,
						px, pz, x0, y0,
						z0, vx0, vy0, vz0,
						ax, ay, az, break_y,
						break_angle, break_length, zone,
						spin_dir, spin_rate, nasty
						)
						VALUES
						($1, $2, $3, $4, $5, $6, $7, $8, $9,
						$10, $11, $12, $13, $14, $15, $16,
						$17, $18, $19, $20,
						$21, $22, $23, $24, $25, $26,
						$27, $28, $29, $30,
						$31, $32, $33, $34,
						$35, $36, $37, $38,
						$39, $40, $41, $42,
						$43, $44, $45,
						$46, $47, $48, $49)`,
						game.ID, game.Year(), inning.Num, half, atBat.Num, atBat.B, atBat.S, atBat.O, nullableString(atBat.StartTFS),
						atBat.Batter, atBat.Stand, atBat.BHeight, atBat.Pitcher, atBat.PThrows, atBat.Des, atBat.Event,
						pitch.Des, pitch.ID, pitch.Type, nullableString(pitch.PitchType), nullableString(pitch.TypeConfidence),
						nullableString(pitch.TFS), pitch.X, pitch.Y, pitch.SvID, nullableString(pitch.StartSpeed), nullableString(pitch.EndSpeed),
						nullableString(pitch.SzTop), nullableString(pitch.SzBottom), nullableString(pitch.PFXX), nullableString(pitch.PFXZ),
						nullableString(pitch.PX), nullableString(pitch.PZ), nullableString(pitch.X0), nullableString(pitch.Y0),
						nullableString(pitch.Z0), nullableString(pitch.VX0), nullableString(pitch.VY0), nullableString(pitch.VZ0),
						nullableString(pitch.AX), nullableString(pitch.AY), nullableString(pitch.AZ), nullableString(pitch.BreakY),
						nullableString(pitch.BreakAngle), nullableString(pitch.BreakLength), nullableString(pitch.Zone),
						nullableString(pitch.SpinDir), nullableString(pitch.SpinRate), nullableString(pitch.Nasty))

					if err != nil {
						if !s.Contains(err.Error(), "duplicate key") {
							log.Fatal(err)
						}
					} else {
						res.Close()
					}
				}
			}
		}
	}
	gamedayapi.FetchByTeamAndYears(teamCode, years, fetchFunction)
}

// ImportPlayersForTeamAndYears saves all player data fields for a team and season.
func ImportPlayersForTeamAndYears(teamCode string, years []int) {
	log.Println("Importing players for " + teamCode)

	// Assumes a pg database exists named go-gameday, a role that can access it.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	issue := db.Ping()
	log.Println(issue)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	fetchFunction := func(game *gamedayapi.Game) {
		log.Println(">>>> " + game.ID + " <<<<")
		var teamID string

		if game.HomeCode == teamCode {
			teamID = game.HomeTeamID
		} else {
			teamID = game.AwayTeamID
		}

		for _, team := range game.Players().Teams {
			for _, player := range team.Players {
				if player.TeamID == teamID {
					res, err := db.Query(`INSERT INTO players
					(id, first, last, num, boxname, rl, bats,
					position, current_position, status, team_abbrev, team_id,
					parent_team_abbrev, parent_team_id, bat_order, game_position, avg,
					hr, rbi, wins, losses, era)
					VALUES
					($1, $2, $3, $4, $5, $6, $7,
					$8, $9, $10, $11, $12,
					$13, $14, $15, $16, $17,
					$18, $19, $20, $21, $22)`,
						player.ID, player.First, player.Last, nullableString(player.Num), player.Boxname, player.Rl, player.Bats,
						player.Position, player.CurrentPosition, player.Status, player.TeamAbbrev, player.TeamID,
						player.ParentTeamAbbrev, nullableString(player.ParentTeamID), nullableString(player.BatOrder), player.GamePosition, player.Avg,
						player.HR, player.RBI, nullableString(player.Wins), nullableString(player.Losses), nullableString(player.ERA))

					if err != nil {
						if !s.Contains(err.Error(), "duplicate key") {
							log.Fatal(err)
						}
					} else {
						res.Close()
					}
				}
			}
		}
	}
	gamedayapi.FetchByTeamAndYears(teamCode, years, fetchFunction)
}

func nullableString(value string) interface{} {
	if value == "" || value == "-" || value == "-.--" {
		return nil
	}
	return value
}
<commit_msg>Relax the constraint on players so we get multiple rows for a player if they played for more than one team.<commit_after>package pggameday

import (
	"database/sql"
	"github.com/ecopony/gamedayapi"
	_ "github.com/lib/pq" // PostgreSQL driver
	"log"
	s "strings"
)

// CreateTables creates the database tables used by the importer. For now it's just the pitches table.
// This will drop tables if they already exist.
func CreateTables() {
	log.Println("Creating database tables.")

	// Assumes a pg database exists named go-gameday, a role that can access it.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	log.Println("\t-Creating pitches table")
	db.Exec("DROP INDEX IF EXISTS pitches_game_id")
	db.Exec("DROP TABLE IF EXISTS pitches")
	db.Exec(`CREATE TABLE pitches (pitchid SERIAL PRIMARY KEY, game_id varchar(40), year int, inning int, half varchar(6),
		at_bat_num int, at_bat_b int, at_bat_s int, at_bat_o int, at_bat_start_tfs int, batter int, stand char(1), b_height
		varchar(4), pitcher int, p_throws char(1), at_bat_des varchar(400), at_bat_event varchar(20), pitch_des varchar(40),
		pitch_id int, pitch_type char(1), pitch_pitch_type char(2), type_confidence DECIMAL(4, 3), pitch_tfs int,
	  	pitch_x DECIMAL(5, 2), pitch_y DECIMAL(5, 2), pitch_sv_id varchar(40), pitch_start_speed DECIMAL(4, 1),
	  	pitch_end_speed DECIMAL(4, 1), sz_top DECIMAL(3, 2), sz_bottom DECIMAL(3, 2), pfx_x DECIMAL(4, 2), pfx_z
	  	DECIMAL(4, 2), px DECIMAL(4, 3), pz DECIMAL(4, 3), x0 DECIMAL(5, 3), y0 DECIMAL(5, 3), z0 DECIMAL(5, 3), vx0
	  	DECIMAL(4, 2), vy0 DECIMAL(6, 3), vz0 DECIMAL(5, 3), ax DECIMAL(5, 3), ay DECIMAL(5, 3), az DECIMAL(5, 3), break_y
	  	DECIMAL(3, 1), break_angle DECIMAL(4, 1), break_length DECIMAL(3, 1), zone int, spin_dir DECIMAL(6, 3), spin_rate
	  	DECIMAL(7, 3), nasty int)`)
	log.Println("\t-Creating pitches index")
	db.Exec("CREATE INDEX pitches_game_id ON pitches (game_id)")
	db.Exec("CREATE UNIQUE INDEX pitches_game_id_pitch_id ON pitches (game_id, pitch_id)")

	log.Println("\t-Creating players table")
	db.Exec("DROP INDEX IF EXISTS players_id")
	db.Exec("DROP TABLE IF EXISTS players")
	db.Exec(`CREATE TABLE players (playerid SERIAL PRIMARY KEY, id int, first varchar(40), last varchar(40), num int,
		boxname varchar(40), rl varchar(1), bats varchar(1), position varchar(2), current_position varchar(2), status
		varchar(1), team_abbrev varchar(3), team_id int, parent_team_abbrev varchar(3), parent_team_id int, bat_order int,
		game_position varchar(2), avg DECIMAL(4, 3), hr int, rbi int, wins int, losses int, era DECIMAL(5, 2))`)
	log.Println("\t-Creating players index")
	db.Exec("CREATE UNIQUE INDEX players_id_team_abbrev ON players (id, team_abbrev)")

	log.Println("Done.")
}

// ImportPitchesForTeamAndYears saves all pitch data fields for a team and season.
func ImportPitchesForTeamAndYears(teamCode string, years []int) {
	log.Println("Importing pitches for " + teamCode)

	// Assumes a pg database exists named go-gameday, a role that can access it.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	issue := db.Ping()
	log.Println(issue)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	fetchFunction := func(game *gamedayapi.Game) {
		log.Println(">>>> " + game.ID + " <<<<")

		var pitchCount int
		err := db.QueryRow("SELECT count(*) FROM pitches WHERE game_id = $1", game.ID).Scan(&pitchCount)
		if err != nil {
			log.Fatal(err)
		}
		if pitchCount > 0 {
			return
		}

		for _, inning := range game.AllInnings().Innings {
			half := "top"
			for _, atBat := range inning.AtBats() {
				if atBat.O == "3" {
					half = "bottom"
				}
				for _, pitch := range atBat.Pitches {
					// log.Println("> " + pitch.SzTop)

					res, err := db.Query(`INSERT INTO pitches
						(game_id, year, inning, half, at_bat_num, at_bat_b, at_bat_s, at_bat_o, at_bat_start_tfs,
						batter, stand, b_height, pitcher, p_throws, at_bat_des, at_bat_event,
						pitch_des, pitch_id, pitch_type, pitch_pitch_type, type_confidence,
						pitch_tfs, pitch_x, pitch_y, pitch_sv_id, pitch_start_speed, pitch_end_speed,
						sz_top, sz_bottom, pfx_x, pfx_z,
						px, pz, x0, y0,
						z0, vx0, vy0, vz0,
						ax, ay, az, break_y,
						break_angle, break_length, zone,
						spin_dir, spin_rate, nasty
						)
						VALUES
						($1, $2, $3, $4, $5, $6, $7, $8, $9,
						$10, $11, $12, $13, $14, $15, $16,
						$17, $18, $19, $20,
						$21, $22, $23, $24, $25, $26,
						$27, $28, $29, $30,
						$31, $32, $33, $34,
						$35, $36, $37, $38,
						$39, $40, $41, $42,
						$43, $44, $45,
						$46, $47, $48, $49)`,
						game.ID, game.Year(), inning.Num, half, atBat.Num, atBat.B, atBat.S, atBat.O, nullableString(atBat.StartTFS),
						atBat.Batter, atBat.Stand, atBat.BHeight, atBat.Pitcher, atBat.PThrows, atBat.Des, atBat.Event,
						pitch.Des, pitch.ID, pitch.Type, nullableString(pitch.PitchType), nullableString(pitch.TypeConfidence),
						nullableString(pitch.TFS), pitch.X, pitch.Y, pitch.SvID, nullableString(pitch.StartSpeed), nullableString(pitch.EndSpeed),
						nullableString(pitch.SzTop), nullableString(pitch.SzBottom), nullableString(pitch.PFXX), nullableString(pitch.PFXZ),
						nullableString(pitch.PX), nullableString(pitch.PZ), nullableString(pitch.X0), nullableString(pitch.Y0),
						nullableString(pitch.Z0), nullableString(pitch.VX0), nullableString(pitch.VY0), nullableString(pitch.VZ0),
						nullableString(pitch.AX), nullableString(pitch.AY), nullableString(pitch.AZ), nullableString(pitch.BreakY),
						nullableString(pitch.BreakAngle), nullableString(pitch.BreakLength), nullableString(pitch.Zone),
						nullableString(pitch.SpinDir), nullableString(pitch.SpinRate), nullableString(pitch.Nasty))

					if err != nil {
						if !s.Contains(err.Error(), "duplicate key") {
							log.Fatal(err)
						}
					} else {
						res.Close()
					}
				}
			}
		}
	}
	gamedayapi.FetchByTeamAndYears(teamCode, years, fetchFunction)
}

// ImportPlayersForTeamAndYears saves all player data fields for a team and season.
func ImportPlayersForTeamAndYears(teamCode string, years []int) {
	log.Println("Importing players for " + teamCode)

	// Assumes a pg database exists named go-gameday, a role that can access it.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	issue := db.Ping()
	log.Println(issue)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	fetchFunction := func(game *gamedayapi.Game) {
		log.Println(">>>> " + game.ID + " <<<<")
		var teamID string

		if game.HomeCode == teamCode {
			teamID = game.HomeTeamID
		} else {
			teamID = game.AwayTeamID
		}

		for _, team := range game.Players().Teams {
			for _, player := range team.Players {
				if player.TeamID == teamID {
					res, err := db.Query(`INSERT INTO players
					(id, first, last, num, boxname, rl, bats,
					position, current_position, status, team_abbrev, team_id,
					parent_team_abbrev, parent_team_id, bat_order, game_position, avg,
					hr, rbi, wins, losses, era)
					VALUES
					($1, $2, $3, $4, $5, $6, $7,
					$8, $9, $10, $11, $12,
					$13, $14, $15, $16, $17,
					$18, $19, $20, $21, $22)`,
						player.ID, player.First, player.Last, nullableString(player.Num), player.Boxname, player.Rl, player.Bats,
						player.Position, player.CurrentPosition, player.Status, player.TeamAbbrev, player.TeamID,
						player.ParentTeamAbbrev, nullableString(player.ParentTeamID), nullableString(player.BatOrder), player.GamePosition, player.Avg,
						player.HR, player.RBI, nullableString(player.Wins), nullableString(player.Losses), nullableString(player.ERA))

					if err != nil {
						if !s.Contains(err.Error(), "duplicate key") {
							log.Fatal(err)
						}
					} else {
						res.Close()
					}
				}
			}
		}
	}
	gamedayapi.FetchByTeamAndYears(teamCode, years, fetchFunction)
}

func nullableString(value string) interface{} {
	if value == "" || value == "-" || value == "-.--" {
		return nil
	}
	return value
}
<|endoftext|>


<commit_before>package pipe

import (
	pipe "."
	"bytes"
	"fmt"
	"os"
	"regexp"
	"time"
)

func Example() {
	err := pipe.Run(
		pipe.Find(pipe.FILES, "."),
		pipe.Grep(`pipe.*\.go$`),
		pipe.WriteLines(os.Stdout),
	)
	fmt.Println("error:", err)
	// Output:
	// pipe.go
	// pipe_test.go
	// error: <nil>
}

func Example_error() {
	counter := func(arg pipe.Arg) error {
		re, err := regexp.Compile("[")
		if err != nil {
			return err
		}
		n := 1
		for s := range arg.In {
			if re.MatchString(s) {
				n++
			}
		}
		arg.Out <- fmt.Sprint(n)
		return nil
	}
	err := pipe.Run(
		pipe.Numbers(1, 100),
		counter,
		pipe.WriteLines(os.Stdout),
	)
	if err == nil {
		fmt.Println("did not catch error")
	}
	// Output:
}

func ExampleSequence() {
	pipe.ForEach(pipe.Sequence(
		pipe.Numbers(1, 25),
		pipe.Grep("3"),
	), func(s string) { fmt.Println(s) })
	// Output:
	// 3
	// 13
	// 23
}

func ExampleForEach() {
	err := pipe.ForEach(pipe.Numbers(1, 5), func(s string) {
		fmt.Print(s)
	})
	if err != nil {
		panic(err)
	}
	// Output:
	// 12345
}

func ExampleOutput() {
	result, err := pipe.Output(pipe.Numbers(1, 3))
	for _, s := range result {
		fmt.Println(s)
	}
	fmt.Println(err)
	// Output:
	// 1
	// 2
	// 3
	// <nil>
}

func ExampleRun() {
	pipe.Run(
		pipe.Echo("line 1", "line 2"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// line 1
	// line 2
}

func ExampleEcho() {
	pipe.Run(
		pipe.Echo("hello", "world"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// hello
	// world
}

func ExampleNumbers() {
	pipe.Run(
		pipe.Numbers(2, 5),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 2
	// 3
	// 4
	// 5
}

func ExampleMap() {
	pipe.Run(
		pipe.Echo("hello", "there", "how", "are", "you?"),
		pipe.Map(func(s string) string {
			return fmt.Sprintf("%d %s", len(s), s)
		}),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleIf() {
	pipe.Run(
		pipe.Numbers(1, 12),
		pipe.If(func(s string) bool { return len(s) > 1 }),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrep() {
	pipe.Run(
		pipe.Numbers(1, 12),
		pipe.Grep(".."),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrepNot() {
	pipe.Run(
		pipe.Numbers(1, 12),
		pipe.GrepNot("^.$"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 10
	// 11
	// 12
}

func ExampleUniq() {
	pipe.Run(
		pipe.Echo("a", "b", "b", "c", "b"),
		pipe.Uniq(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// a
	// b
	// c
	// b
}

func ExampleUniqWithCount() {
	pipe.Run(
		pipe.Echo("a", "b", "b", "c"),
		pipe.UniqWithCount(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1 a
	// 2 b
	// 1 c
}

func ExampleParallelMap() {
	pipe.Run(
		pipe.Echo("hello", "there", "how", "are", "you?"),
		pipe.ParallelMap(4, func(s string) string {
			// Sleep some amount to ensure that ParalellMap
			// implementation handles out of order results.
			time.Sleep(10 * time.Duration(len(s)) * time.Millisecond)
			return fmt.Sprintf("%d %s", len(s), s)
		}),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleSubstitute() {
	pipe.Run(
		pipe.Numbers(1, 5),
		pipe.Substitute("(3)", "$1$1"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 33
	// 4
	// 5
}

func ExampleNumeric() {
	pipe.Run(
		pipe.Echo(
			"a 100",
			"b 20",
			"c notanumber", // Will sort last since column 2 is not a number
			"d",            // Will sort earliest since column 2 is missing
		),
		pipe.Sort(pipe.Numeric(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// d
	// b 20
	// a 100
	// c notanumber
}

func ExampleTextual() {
	pipe.Run(
		pipe.Echo(
			"10 bananas",
			"20 apples",
			"30", // Will sort first since column 2 is missing
		),
		pipe.Sort(pipe.Textual(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 30
	// 20 apples
	// 10 bananas
}

func ExampleDescending() {
	pipe.Run(
		pipe.Echo(
			"100",
			"20",
			"50",
		),
		pipe.Sort(pipe.Descending(pipe.Numeric(1))),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 100
	// 50
	// 20
}

func ExampleSort() {
	pipe.Run(
		pipe.Echo("banana", "apple", "cheese", "apple"),
		pipe.Sort(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// apple
	// apple
	// banana
	// cheese
}
func ExampleSort_twoTextColumns() {
	pipe.Run(
		pipe.Echo(
			"2 green bananas",
			"3 red apples",
			"4 yellow bananas",
			"5 brown pears",
			"6 green apples",
		),
		pipe.Sort(pipe.Textual(2), pipe.Textual(3)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 5 brown pears
	// 6 green apples
	// 2 green bananas
	// 3 red apples
	// 4 yellow bananas
}

func ExampleSort_twoNumericColumns() {
	pipe.Run(
		pipe.Echo(
			"1970 12",
			"1970 6",
			"1950 6",
			"1980 9",
		),
		pipe.Sort(pipe.Numeric(1), pipe.Numeric(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1950 6
	// 1970 6
	// 1970 12
	// 1980 9
}

func ExampleSort_mixedColumns() {
	pipe.Run(
		pipe.Echo(
			"1970 march",
			"1970 feb",
			"1950 june",
			"1980 sep",
		),
		pipe.Sort(pipe.Numeric(1), pipe.Textual(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1950 june
	// 1970 feb
	// 1970 march
	// 1980 sep
}

func ExampleReverse() {
	pipe.Run(
		pipe.Echo("a", "b"),
		pipe.Reverse(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// b
	// a
}

func ExampleSample() {
	pipe.Run(
		pipe.Numbers(100, 200),
		pipe.Sample(4),
		pipe.WriteLines(os.Stdout),
	)
	// Output not checked since it is non-deterministic.
}

func ExampleFirst() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.First(3),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 3
}

func ExampleLast() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.Last(2),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 9
	// 10
}

func ExampleDropFirst() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.DropFirst(8),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 9
	// 10
}

func ExampleDropLast() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.DropLast(3),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 3
	// 4
	// 5
	// 6
	// 7
}

func ExampleNumberLines() {
	pipe.Run(
		pipe.Echo("a", "b"),
		pipe.NumberLines(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	//     1 a
	//     2 b
}

func ExampleSlice() {
	pipe.Run(
		pipe.Echo("hello", "world."),
		pipe.Slice(2, 5),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// llo
	// rld
}

func ExampleColumns() {
	pipe.Run(
		pipe.Echo("hello world"),
		pipe.Columns(2, 3, 1),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// world hello
}

func ExampleFind() {
	pipe.Run(
		pipe.Find(pipe.FILES, "."),
		pipe.Grep("pipe"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// pipe.go
	// pipe_test.go
}

func ExampleFind_dirs() {
	pipe.Run(
		pipe.Find(pipe.DIRS, "."),
		pipe.GrepNot("git"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// .
}

func ExampleFind_error() {
	err := pipe.Run(pipe.Find(pipe.ALL, "/no_such_dir"))
	if err == nil {
		fmt.Println("pipe.Find did not return expected error")
	}
	// Output:
}

func ExampleCat() {
	pipe.Run(
		pipe.Cat("pipe_test.go"),
		pipe.Grep("^func ExampleCat"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// func ExampleCat() {
}

func ExampleWriteLines() {
	pipe.Run(
		pipe.Numbers(1, 3),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 3
}

func ExampleReadLines() {
	pipe.Run(
		pipe.ReadLines(bytes.NewBufferString("the\nquick\nbrown\nfox\n")),
		pipe.Sort(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// brown
	// fox
	// quick
	// the
}

func ExampleCommand() {
	pipe.Run(
		pipe.Numbers(1, 100),
		pipe.Command("wc", "-l"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 100
}

func ExampleCommand_outputOnly() {
	pipe.Run(
		pipe.Command("find", ".", "-type", "f", "-print"),
		pipe.Grep(`^\./pipe.*\.go$`),
		pipe.Sort(),
		pipe.WriteLines(os.Stdout),
	)

	// Output:
	// ./pipe.go
	// ./pipe_test.go
}

func ExampleCommand_withError() {
	err := pipe.Run(pipe.Command("no_such_command"))
	if err == nil {
		fmt.Println("execution of missing command succeeded unexpectedly")
	}
	// Output:
}
<commit_msg>switched to importing from github path<commit_after>package pipe_test

import (
	"github.com/ghemawat/pipe"

	"bytes"
	"fmt"
	"os"
	"regexp"
	"time"
)

func Example() {
	err := pipe.Run(
		pipe.Find(pipe.FILES, "."),
		pipe.Grep(`pipe.*\.go$`),
		pipe.WriteLines(os.Stdout),
	)
	fmt.Println("error:", err)
	// Output:
	// pipe.go
	// pipe_test.go
	// error: <nil>
}

func Example_error() {
	counter := func(arg pipe.Arg) error {
		re, err := regexp.Compile("[")
		if err != nil {
			return err
		}
		n := 1
		for s := range arg.In {
			if re.MatchString(s) {
				n++
			}
		}
		arg.Out <- fmt.Sprint(n)
		return nil
	}
	err := pipe.Run(
		pipe.Numbers(1, 100),
		counter,
		pipe.WriteLines(os.Stdout),
	)
	if err == nil {
		fmt.Println("did not catch error")
	}
	// Output:
}

func ExampleSequence() {
	pipe.ForEach(pipe.Sequence(
		pipe.Numbers(1, 25),
		pipe.Grep("3"),
	), func(s string) { fmt.Println(s) })
	// Output:
	// 3
	// 13
	// 23
}

func ExampleForEach() {
	err := pipe.ForEach(pipe.Numbers(1, 5), func(s string) {
		fmt.Print(s)
	})
	if err != nil {
		panic(err)
	}
	// Output:
	// 12345
}

func ExampleOutput() {
	result, err := pipe.Output(pipe.Numbers(1, 3))
	for _, s := range result {
		fmt.Println(s)
	}
	fmt.Println(err)
	// Output:
	// 1
	// 2
	// 3
	// <nil>
}

func ExampleRun() {
	pipe.Run(
		pipe.Echo("line 1", "line 2"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// line 1
	// line 2
}

func ExampleEcho() {
	pipe.Run(
		pipe.Echo("hello", "world"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// hello
	// world
}

func ExampleNumbers() {
	pipe.Run(
		pipe.Numbers(2, 5),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 2
	// 3
	// 4
	// 5
}

func ExampleMap() {
	pipe.Run(
		pipe.Echo("hello", "there", "how", "are", "you?"),
		pipe.Map(func(s string) string {
			return fmt.Sprintf("%d %s", len(s), s)
		}),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleIf() {
	pipe.Run(
		pipe.Numbers(1, 12),
		pipe.If(func(s string) bool { return len(s) > 1 }),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrep() {
	pipe.Run(
		pipe.Numbers(1, 12),
		pipe.Grep(".."),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrepNot() {
	pipe.Run(
		pipe.Numbers(1, 12),
		pipe.GrepNot("^.$"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 10
	// 11
	// 12
}

func ExampleUniq() {
	pipe.Run(
		pipe.Echo("a", "b", "b", "c", "b"),
		pipe.Uniq(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// a
	// b
	// c
	// b
}

func ExampleUniqWithCount() {
	pipe.Run(
		pipe.Echo("a", "b", "b", "c"),
		pipe.UniqWithCount(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1 a
	// 2 b
	// 1 c
}

func ExampleParallelMap() {
	pipe.Run(
		pipe.Echo("hello", "there", "how", "are", "you?"),
		pipe.ParallelMap(4, func(s string) string {
			// Sleep some amount to ensure that ParalellMap
			// implementation handles out of order results.
			time.Sleep(10 * time.Duration(len(s)) * time.Millisecond)
			return fmt.Sprintf("%d %s", len(s), s)
		}),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleSubstitute() {
	pipe.Run(
		pipe.Numbers(1, 5),
		pipe.Substitute("(3)", "$1$1"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 33
	// 4
	// 5
}

func ExampleNumeric() {
	pipe.Run(
		pipe.Echo(
			"a 100",
			"b 20",
			"c notanumber", // Will sort last since column 2 is not a number
			"d",            // Will sort earliest since column 2 is missing
		),
		pipe.Sort(pipe.Numeric(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// d
	// b 20
	// a 100
	// c notanumber
}

func ExampleTextual() {
	pipe.Run(
		pipe.Echo(
			"10 bananas",
			"20 apples",
			"30", // Will sort first since column 2 is missing
		),
		pipe.Sort(pipe.Textual(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 30
	// 20 apples
	// 10 bananas
}

func ExampleDescending() {
	pipe.Run(
		pipe.Echo(
			"100",
			"20",
			"50",
		),
		pipe.Sort(pipe.Descending(pipe.Numeric(1))),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 100
	// 50
	// 20
}

func ExampleSort() {
	pipe.Run(
		pipe.Echo("banana", "apple", "cheese", "apple"),
		pipe.Sort(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// apple
	// apple
	// banana
	// cheese
}
func ExampleSort_twoTextColumns() {
	pipe.Run(
		pipe.Echo(
			"2 green bananas",
			"3 red apples",
			"4 yellow bananas",
			"5 brown pears",
			"6 green apples",
		),
		pipe.Sort(pipe.Textual(2), pipe.Textual(3)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 5 brown pears
	// 6 green apples
	// 2 green bananas
	// 3 red apples
	// 4 yellow bananas
}

func ExampleSort_twoNumericColumns() {
	pipe.Run(
		pipe.Echo(
			"1970 12",
			"1970 6",
			"1950 6",
			"1980 9",
		),
		pipe.Sort(pipe.Numeric(1), pipe.Numeric(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1950 6
	// 1970 6
	// 1970 12
	// 1980 9
}

func ExampleSort_mixedColumns() {
	pipe.Run(
		pipe.Echo(
			"1970 march",
			"1970 feb",
			"1950 june",
			"1980 sep",
		),
		pipe.Sort(pipe.Numeric(1), pipe.Textual(2)),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1950 june
	// 1970 feb
	// 1970 march
	// 1980 sep
}

func ExampleReverse() {
	pipe.Run(
		pipe.Echo("a", "b"),
		pipe.Reverse(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// b
	// a
}

func ExampleSample() {
	pipe.Run(
		pipe.Numbers(100, 200),
		pipe.Sample(4),
		pipe.WriteLines(os.Stdout),
	)
	// Output not checked since it is non-deterministic.
}

func ExampleFirst() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.First(3),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 3
}

func ExampleLast() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.Last(2),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 9
	// 10
}

func ExampleDropFirst() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.DropFirst(8),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 9
	// 10
}

func ExampleDropLast() {
	pipe.Run(
		pipe.Numbers(1, 10),
		pipe.DropLast(3),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 3
	// 4
	// 5
	// 6
	// 7
}

func ExampleNumberLines() {
	pipe.Run(
		pipe.Echo("a", "b"),
		pipe.NumberLines(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	//     1 a
	//     2 b
}

func ExampleSlice() {
	pipe.Run(
		pipe.Echo("hello", "world."),
		pipe.Slice(2, 5),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// llo
	// rld
}

func ExampleColumns() {
	pipe.Run(
		pipe.Echo("hello world"),
		pipe.Columns(2, 3, 1),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// world hello
}

func ExampleFind() {
	pipe.Run(
		pipe.Find(pipe.FILES, "."),
		pipe.Grep("pipe"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// pipe.go
	// pipe_test.go
}

func ExampleFind_dirs() {
	pipe.Run(
		pipe.Find(pipe.DIRS, "."),
		pipe.GrepNot("git"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// .
}

func ExampleFind_error() {
	err := pipe.Run(pipe.Find(pipe.ALL, "/no_such_dir"))
	if err == nil {
		fmt.Println("pipe.Find did not return expected error")
	}
	// Output:
}

func ExampleCat() {
	pipe.Run(
		pipe.Cat("pipe_test.go"),
		pipe.Grep("^func ExampleCat"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// func ExampleCat() {
}

func ExampleWriteLines() {
	pipe.Run(
		pipe.Numbers(1, 3),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 1
	// 2
	// 3
}

func ExampleReadLines() {
	pipe.Run(
		pipe.ReadLines(bytes.NewBufferString("the\nquick\nbrown\nfox\n")),
		pipe.Sort(),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// brown
	// fox
	// quick
	// the
}

func ExampleCommand() {
	pipe.Run(
		pipe.Numbers(1, 100),
		pipe.Command("wc", "-l"),
		pipe.WriteLines(os.Stdout),
	)
	// Output:
	// 100
}

func ExampleCommand_outputOnly() {
	pipe.Run(
		pipe.Command("find", ".", "-type", "f", "-print"),
		pipe.Grep(`^\./pipe.*\.go$`),
		pipe.Sort(),
		pipe.WriteLines(os.Stdout),
	)

	// Output:
	// ./pipe.go
	// ./pipe_test.go
}

func ExampleCommand_withError() {
	err := pipe.Run(pipe.Command("no_such_command"))
	if err == nil {
		fmt.Println("execution of missing command succeeded unexpectedly")
	}
	// Output:
}
<|endoftext|>