<commit_before>package frame

import "bytes"

import "errors"
import "io"
import "sync"
import (
	"ginkp/command"
)

var (
	// ErrFrameIsNotCommand ...
	ErrFrameIsNotCommand = errors.New("frame isn't command")
	// ErrEmptyFrame ...
	ErrEmptyFrame = errors.New("empty frame")
	framePool     = sync.Pool{
		New: func() interface{} {
			return &Frame{}
		},
	}
)

// Frame ...
// Pinkb protocol frame.
// The frame header contains two bytes defining the type and the length (in bytes) of the data that follows the
// header. If the most significant bit of the header is reset, then all the data received with the frame should be
// appended to the current file being received if the file has been opened, otherwise the data should be
// discarded. If the bit is set, the data should be considered as a command changing the protocol state. The first
// data byte in the frame is the command number. The rest of the bytes form an argument. A command
// argument is an arbitrary character set not required to be bounded with a '\0'.
//
//   +-+-------+--------+--------------+
//   |1|  HI  0   LO   1|     DATA     |
//   +-+-------+--------+--------------+
//   |                  | <- 32k max-> |
//   |<-    2 bytes   ->|              + datablock/msg's argument
//                      + length of the frame without the header  1 byte
//    + this is a command
type Frame struct {
	bytes.Buffer
}

// NewFrame ...
// New binkp frame from frame pool.
func NewFrame() *Frame {
	return framePool.Get().(*Frame)
}

// DeleteFrame ...
// Resets and returns frame to frame pool for reusing in the future
func DeleteFrame(frame *Frame) {
	frame.Reset()
	framePool.Put(frame)
}

// IsCommand ...
func (frame *Frame) IsCommand() bool {
	if frame.Len() < 1 {
		frame.Grow(1)
		return false
	}
	return frame.Bytes()[0]&0x80>>7 == 1
}

// UpdateLen ...
// Recalcultes and updates len field.
func (frame *Frame) UpdateLen() {
	fLen := uint16(frame.Len())
	fBytes := frame.Bytes()
	if fLen > 2 {
		fBytes[0] = (byte(fLen&0xFF) & 0x40) | (fBytes[0] & 0x80)
		fBytes[1] = byte(fLen >> 8)
	} else {
		frame.Grow(2 - int(fLen))
	}
}

// prepare ...
// Does preparations to write command code.
// Frows frame or truncate if need
func (frame *Frame) prepare() {
	fLen := frame.Len()
	if fLen < 3 {
		frame.Grow(3 - fLen)
	} else if fLen > 3 {
		frame.Truncate(fLen - 3)
	}
}

func (frame *Frame) setCommandFlag(command func(*Frame)) {
	frame.prepare()
	frame.Bytes()[0] = frame.Bytes()[0] | 0x80
	command(frame)
}

func (frame *Frame) writeCommandWithArgs(command command.Code, args []byte) {
	fBytes := frame.Bytes()
	frame.prepare()
	fBytes[2] = byte(command)
	_, err := frame.Write(args)
	if err != nil {
		panic(err)
	}
	frame.UpdateLen()
}

func (frame *Frame) writeCommand(command command.Code) {
	fBytes := frame.Bytes()
	frame.prepare()
	fBytes[2] = byte(command)
	frame.UpdateLen()
}

// SetNUL ...
// The command argument is ignored (and is possibly logged).
// This is the way we transmit the nodelist information,
// the sysop’s name and so on.
// eg, "ZYZ Dima Maloff"
func (frame *Frame) SetNUL(msg string) {
	frame.writeCommandWithArgs(command.M_NUL, []byte(msg))
}

// SetADR ...
// A list of 5D addresses delimited by spaces.
// eg, "2:5047/13@fidonet 2:5047/0@fidonet"
func (frame *Frame) SetADR(addr []string) {
	frame.writeCommand(command.M_ADR)
	for _, address := range addr {
		_, err := frame.WriteString(address + " ")
		assert(err)
	}
}

// SetPWD ...
// A password. After the successful processing of the password received from the remote,
// the binkd server rescans the queue.
// eg, "pAsSwOrD"
func (frame *Frame) SetPWD(password string) {
	frame.writeCommandWithArgs(command.M_PWD, []byte(password))
}

// SetFILE ...
// The properties of the next file. They are delimited by spaces: filename without spaces, size, UNIX
// time, the offset to transfer the file. All the numbers are decimal. All the data blocks received after that
// relate to this file until the next M_FILE is received. There is no special endoffile marker since the file
// size is known beforehand. Binkd will append the "excessive" blocks to the current file. We start
// transmitting every new file from the offset 0. On receiving M_GET from the remote system we must
// do the seek operation.
// eg, "config.sys 125 2476327846 0"
func (frame *Frame) SetFILE(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.String()))
}

// SetOK ...
// A reply to the correct password. The binkd client rescans the queue after receiving the message.
// The command argument is ignored.
func (frame *Frame) SetOK() {
	frame.writeCommand(command.M_OK)
}

// SetEOB ...
// EndofBatch. EOB is transmitted after all the files have been sent. If we are in the EOB state (all the
// files are sent), we get EOB from the remote (no more files for us), we received all acknowledgements
// for all the sent files, we received all the files resent in reply to GET,
// then the session is considered to be successfully completed.
func (frame *Frame) SetEOB() {
	frame.writeCommand(command.M_EOB)
}

// SetGOT ...
// It is sent as an acknowledgement by the system which has received a file after receiving the last
// portion of the file data. The arguments are copies of the FILE command arguments received from the
// remote system except the last one, the offset which should not be returned to the system which sent
// M_FILE. GOT may also be sent during the process of receiving a file; the sending partner should react
// to it with the destructive skip.
// eg, "config.sys 125 2476327846"
func (frame *Frame) SetGOT(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.StringWithoutOff()))
}

// SetERR ...
// A fatal error. The partner who has sent M_ERR aborts the session. The argument contains the text
// explaining the reason and it is logged. Binkd sends M_ERR in response to an incorrect password.
// eg, "Incorrect password"
func (frame *Frame) SetERR(err error) {
	frame.writeCommandWithArgs(command.M_ERR, []byte(err.Error()))
}

// SetBSY ...
// Our system sends it if it is busy. The receiving partner ignores the argument (logs it).
// eg, "Too many servers are running already"
func (frame *Frame) SetBSY(err error) {
	frame.writeCommandWithArgs(command.M_ERR, []byte(err.Error()))
}

// SetGET ...
// M_GET is used as a request to resend a file. The M_GET arguments copy the arguments of the
// M_FILE command which we’d like to see from the remote system. :) Binkd sends it as a response to
// M_FILE if it does not like the offset from which the file transmission has been started by the remote
// system.
// eg, "config.sys 125 2476327846 100"
// At present binkd handles it as follows: according to the first fields (name/size/UNIXtime) it
// determines whether the M_GET argument is the file we currently transmit (or the file has been
// transmitted and we are waiting for M_GOT for it). If this is the case it seeks the specified offset in the
// file and sends M_FILE after that. For the example above M_FILE will have the following arguments:
// "config.sys 125 2476327846 100"
func (frame *Frame) SetGET(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.String()))
}

// SetSKIP ...
// Non destructive skip. An example of the argument line:
// "config.sys 125 2476327846"
func (frame *Frame) SetSKIP(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.StringWithoutOff()))
}

// CommandCode ...
// Returns command.Code or error ErrFrameIsNotCommand,
// if frame doesn't contain command or
// ErrEmptyFrame, if frame is empty (less 3 bytes)
func (frame *Frame) CommandCode() (command.Code, error) {
	fLen := frame.Len()
	fBytes := frame.Bytes()
	if fLen >= 3 {
		if (fBytes[0]&0x80)>>1 != 1 {
			return 0, ErrFrameIsNotCommand
		}
		return command.Code(fBytes[2]), nil
	}
	return 0, ErrEmptyFrame
}

// Args ...
// Returns string, which represents command args or error
// ErrFrameIsNotCommand, if frame doesn't contain command or
// ErrEmptyFrame if frame is empty (less 3 bytes)
func (frame *Frame) Args() (string, error) {
	fLen := frame.Len()
	fBytes := frame.Bytes()
	if fLen >= 3 {
		if (fBytes[0]&0x80)>>1 != 1 {
			return "", ErrFrameIsNotCommand
		}
		return string(fBytes[3:]), nil
	}
	return "", ErrEmptyFrame
}

// WriteDataTo ...
// Write bytes from frame data section to io.Writer.
// Returns error if occcures.
func (frame *Frame) WriteDataTo(w io.Writer) (int64, error) {
	if frame.Len() < 2 {
		return 0, ErrEmptyFrame
	}
	buf := bytes.NewBuffer(frame.Bytes()[2:])
	n, err := buf.WriteTo(w)
	frame.UpdateLen()
	return n, err
}

// ReadDataFrom ...
// Read bytes from io.Reader to frame data section.
// Returns error if occcures.
func (frame *Frame) ReadDataFrom(r io.Reader) (int64, error) {
	frame.prepare()
	n, err := frame.ReadFrom(r)
	frame.UpdateLen()
	return n, err
}

// Copy ...
// Makes full frame copy.
func (frame *Frame) Copy() (*Frame, error) {
	copyFrame := NewFrame()
	_, err := copyFrame.ReadFrom(frame)
	return copyFrame, err
}

// Reader ...
// Returns io.Reader, which reads bytes from frame(including header).
// Doesn't copy frame, so all changes over original frame
// will be reflected in io.Reader data
func (frame *Frame) Reader() io.Reader {
	return bytes.NewReader(frame.Bytes())
}

// DataReader ...
// Returns io.Reader, which reads bytes from frame data section
// Doesn't copy frame data, so all changes over original frame
// will be reflected in io.Reader data
func (frame *Frame) DataReader() (io.Reader, error) {
	fLen := frame.Len()
	if fLen >= 3 {
		return bytes.NewReader(frame.Bytes()[3:]), nil
	}
	return nil, ErrEmptyFrame
}

// DataSize ...
// Returns frame data section size (0 if frame is less then 2 bytes)
func (frame *Frame) DataSize() int {
	fLen := frame.Len()
	if fLen >= 2 {
		return fLen - 2
	}
	return 0
}
<commit_msg>specification url<commit_after>package frame

// binkp specification: http://ftsc.org/docs/fts-1026.001

import "bytes"
import "errors"
import "io"
import "sync"
import (
	"ginkp/command"
)

var (
	// ErrFrameIsNotCommand ...
	ErrFrameIsNotCommand = errors.New("frame isn't command")
	// ErrEmptyFrame ...
	ErrEmptyFrame = errors.New("empty frame")
	framePool     = sync.Pool{
		New: func() interface{} {
			return &Frame{}
		},
	}
)

// Frame ...
// Pinkb protocol frame.
// The frame header contains two bytes defining the type and the length (in bytes) of the data that follows the
// header. If the most significant bit of the header is reset, then all the data received with the frame should be
// appended to the current file being received if the file has been opened, otherwise the data should be
// discarded. If the bit is set, the data should be considered as a command changing the protocol state. The first
// data byte in the frame is the command number. The rest of the bytes form an argument. A command
// argument is an arbitrary character set not required to be bounded with a '\0'.
//
//   +-+-------+--------+--------------+
//   |1|  HI  0   LO   1|     DATA     |
//   +-+-------+--------+--------------+
//   |                  | <- 32k max-> |
//   |<-    2 bytes   ->|              + datablock/msg's argument
//                      + length of the frame without the header  1 byte
//    + this is a command
type Frame struct {
	bytes.Buffer
}

// NewFrame ...
// New binkp frame from frame pool.
func NewFrame() *Frame {
	return framePool.Get().(*Frame)
}

// DeleteFrame ...
// Resets and returns frame to frame pool for reusing in the future
func DeleteFrame(frame *Frame) {
	frame.Reset()
	framePool.Put(frame)
}

// IsCommand ...
func (frame *Frame) IsCommand() bool {
	if frame.Len() < 1 {
		frame.Grow(1)
		return false
	}
	return frame.Bytes()[0]&0x80>>7 == 1
}

// UpdateLen ...
// Recalcultes and updates len field.
func (frame *Frame) UpdateLen() {
	fLen := uint16(frame.Len())
	fBytes := frame.Bytes()
	if fLen > 2 {
		fBytes[0] = (byte(fLen&0xFF) & 0x40) | (fBytes[0] & 0x80)
		fBytes[1] = byte(fLen >> 8)
	} else {
		frame.Grow(2 - int(fLen))
	}
}

// prepare ...
// Does preparations to write command code.
// Frows frame or truncate if need
func (frame *Frame) prepare() {
	fLen := frame.Len()
	if fLen < 3 {
		frame.Grow(3 - fLen)
	} else if fLen > 3 {
		frame.Truncate(fLen - 3)
	}
}

func (frame *Frame) setCommandFlag(command func(*Frame)) {
	frame.prepare()
	frame.Bytes()[0] = frame.Bytes()[0] | 0x80
	command(frame)
}

func (frame *Frame) writeCommandWithArgs(command command.Code, args []byte) {
	fBytes := frame.Bytes()
	frame.prepare()
	fBytes[2] = byte(command)
	_, err := frame.Write(args)
	if err != nil {
		panic(err)
	}
	frame.UpdateLen()
}

func (frame *Frame) writeCommand(command command.Code) {
	fBytes := frame.Bytes()
	frame.prepare()
	fBytes[2] = byte(command)
	frame.UpdateLen()
}

// SetNUL ...
// The command argument is ignored (and is possibly logged).
// This is the way we transmit the nodelist information,
// the sysop’s name and so on.
// eg, "ZYZ Dima Maloff"
func (frame *Frame) SetNUL(msg string) {
	frame.writeCommandWithArgs(command.M_NUL, []byte(msg))
}

// SetADR ...
// A list of 5D addresses delimited by spaces.
// eg, "2:5047/13@fidonet 2:5047/0@fidonet"
func (frame *Frame) SetADR(addr []string) {
	frame.writeCommand(command.M_ADR)
	for _, address := range addr {
		_, err := frame.WriteString(address + " ")
		assert(err)
	}
}

// SetPWD ...
// A password. After the successful processing of the password received from the remote,
// the binkd server rescans the queue.
// eg, "pAsSwOrD"
func (frame *Frame) SetPWD(password string) {
	frame.writeCommandWithArgs(command.M_PWD, []byte(password))
}

// SetFILE ...
// The properties of the next file. They are delimited by spaces: filename without spaces, size, UNIX
// time, the offset to transfer the file. All the numbers are decimal. All the data blocks received after that
// relate to this file until the next M_FILE is received. There is no special endoffile marker since the file
// size is known beforehand. Binkd will append the "excessive" blocks to the current file. We start
// transmitting every new file from the offset 0. On receiving M_GET from the remote system we must
// do the seek operation.
// eg, "config.sys 125 2476327846 0"
func (frame *Frame) SetFILE(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.String()))
}

// SetOK ...
// A reply to the correct password. The binkd client rescans the queue after receiving the message.
// The command argument is ignored.
func (frame *Frame) SetOK() {
	frame.writeCommand(command.M_OK)
}

// SetEOB ...
// EndofBatch. EOB is transmitted after all the files have been sent. If we are in the EOB state (all the
// files are sent), we get EOB from the remote (no more files for us), we received all acknowledgements
// for all the sent files, we received all the files resent in reply to GET,
// then the session is considered to be successfully completed.
func (frame *Frame) SetEOB() {
	frame.writeCommand(command.M_EOB)
}

// SetGOT ...
// It is sent as an acknowledgement by the system which has received a file after receiving the last
// portion of the file data. The arguments are copies of the FILE command arguments received from the
// remote system except the last one, the offset which should not be returned to the system which sent
// M_FILE. GOT may also be sent during the process of receiving a file; the sending partner should react
// to it with the destructive skip.
// eg, "config.sys 125 2476327846"
func (frame *Frame) SetGOT(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.StringWithoutOff()))
}

// SetERR ...
// A fatal error. The partner who has sent M_ERR aborts the session. The argument contains the text
// explaining the reason and it is logged. Binkd sends M_ERR in response to an incorrect password.
// eg, "Incorrect password"
func (frame *Frame) SetERR(err error) {
	frame.writeCommandWithArgs(command.M_ERR, []byte(err.Error()))
}

// SetBSY ...
// Our system sends it if it is busy. The receiving partner ignores the argument (logs it).
// eg, "Too many servers are running already"
func (frame *Frame) SetBSY(err error) {
	frame.writeCommandWithArgs(command.M_ERR, []byte(err.Error()))
}

// SetGET ...
// M_GET is used as a request to resend a file. The M_GET arguments copy the arguments of the
// M_FILE command which we’d like to see from the remote system. :) Binkd sends it as a response to
// M_FILE if it does not like the offset from which the file transmission has been started by the remote
// system.
// eg, "config.sys 125 2476327846 100"
// At present binkd handles it as follows: according to the first fields (name/size/UNIXtime) it
// determines whether the M_GET argument is the file we currently transmit (or the file has been
// transmitted and we are waiting for M_GOT for it). If this is the case it seeks the specified offset in the
// file and sends M_FILE after that. For the example above M_FILE will have the following arguments:
// "config.sys 125 2476327846 100"
func (frame *Frame) SetGET(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.String()))
}

// SetSKIP ...
// Non destructive skip. An example of the argument line:
// "config.sys 125 2476327846"
func (frame *Frame) SetSKIP(file *command.FileConfig) {
	frame.writeCommandWithArgs(command.M_FILE, []byte(file.StringWithoutOff()))
}

// CommandCode ...
// Returns command.Code or error ErrFrameIsNotCommand,
// if frame doesn't contain command or
// ErrEmptyFrame, if frame is empty (less 3 bytes)
func (frame *Frame) CommandCode() (command.Code, error) {
	fLen := frame.Len()
	fBytes := frame.Bytes()
	if fLen >= 3 {
		if (fBytes[0]&0x80)>>1 != 1 {
			return 0, ErrFrameIsNotCommand
		}
		return command.Code(fBytes[2]), nil
	}
	return 0, ErrEmptyFrame
}

// Args ...
// Returns string, which represents command args or error
// ErrFrameIsNotCommand, if frame doesn't contain command or
// ErrEmptyFrame if frame is empty (less 3 bytes)
func (frame *Frame) Args() (string, error) {
	fLen := frame.Len()
	fBytes := frame.Bytes()
	if fLen >= 3 {
		if (fBytes[0]&0x80)>>1 != 1 {
			return "", ErrFrameIsNotCommand
		}
		return string(fBytes[3:]), nil
	}
	return "", ErrEmptyFrame
}

// WriteDataTo ...
// Write bytes from frame data section to io.Writer.
// Returns error if occcures.
func (frame *Frame) WriteDataTo(w io.Writer) (int64, error) {
	if frame.Len() < 2 {
		return 0, ErrEmptyFrame
	}
	buf := bytes.NewBuffer(frame.Bytes()[2:])
	n, err := buf.WriteTo(w)
	frame.UpdateLen()
	return n, err
}

// ReadDataFrom ...
// Read bytes from io.Reader to frame data section.
// Returns error if occcures.
func (frame *Frame) ReadDataFrom(r io.Reader) (int64, error) {
	frame.prepare()
	n, err := frame.ReadFrom(r)
	frame.UpdateLen()
	return n, err
}

// Copy ...
// Makes full frame copy.
func (frame *Frame) Copy() (*Frame, error) {
	copyFrame := NewFrame()
	_, err := copyFrame.ReadFrom(frame)
	return copyFrame, err
}

// Reader ...
// Returns io.Reader, which reads bytes from frame(including header).
// Doesn't copy frame, so all changes over original frame
// will be reflected in io.Reader data
func (frame *Frame) Reader() io.Reader {
	return bytes.NewReader(frame.Bytes())
}

// DataReader ...
// Returns io.Reader, which reads bytes from frame data section
// Doesn't copy frame data, so all changes over original frame
// will be reflected in io.Reader data
func (frame *Frame) DataReader() (io.Reader, error) {
	fLen := frame.Len()
	if fLen >= 3 {
		return bytes.NewReader(frame.Bytes()[3:]), nil
	}
	return nil, ErrEmptyFrame
}

// DataSize ...
// Returns frame data section size (0 if frame is less then 2 bytes)
func (frame *Frame) DataSize() int {
	fLen := frame.Len()
	if fLen >= 2 {
		return fLen - 2
	}
	return 0
}
<|endoftext|>


<commit_before>package approvaltests

import (
	"fmt"
	"strings"

	"reflect"
)

type emptyType struct{}

var (
	// SkipThisCombination should be returned if you do not want to process a particular combination
	SkipThisCombination = "♬ SKIP THIS COMBINATION ♬"

	empty           = emptyType{}
	emptyCollection = []emptyType{empty}
)

// VerifyAllCombinationsFor1 Example:
//   VerifyAllCombinationsFor1(t, "uppercase", func(x interface{}) string { return strings.ToUpper(x.(string)) }, []string("dog", "cat"})
func VerifyAllCombinationsFor1(t Failable, header string, transform func(interface{}) string, collection1 interface{}) error {
	transform2 := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1)
	}

	return VerifyAllCombinationsFor9(t, header, transform2, collection1,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor2 calls the transform function with all combinations
// from collection 1 and collection 2. The resulting received file contains all
// inputs and the resulting outputs. The received file is then compared to the
// approved version. If the transform function returns SkipThisCombination the
// output of this combination won't be displayed inside the received file.
func VerifyAllCombinationsFor2(t Failable, header string, transform func(interface{}, interface{}) string, collection1 interface{}, collection2 interface{}) error {
	transform2 := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2)
	}

	return VerifyAllCombinationsFor9(t, header, transform2, collection1,
		collection2,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor3 is for combinations of 3.
func VerifyAllCombinationsFor3(
	t Failable,
	header string,
	transform func(p1, p2, p3 interface{}) string,
	collection1, collection2, collection3 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor4 is for combinations of 4.
func VerifyAllCombinationsFor4(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4 interface{}) string,
	collection1, collection2, collection3, collection4 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor5 is for combinations of 5.
func VerifyAllCombinationsFor5(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5 interface{}) string,
	collection1, collection2, collection3, collection4, collection5 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor6 is for combinations of 6.
func VerifyAllCombinationsFor6(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5, p6 interface{}) string,
	collection1, collection2, collection3, collection4, collection5, collection6 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5, p6)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		collection6,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor7 is for combinations of 7.
func VerifyAllCombinationsFor7(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5, p6, p7 interface{}) string,
	collection1, collection2, collection3, collection4, collection5, collection6, collection7 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5, p6, p7)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		collection6,
		collection7,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor8 is for combinations of 8.
func VerifyAllCombinationsFor8(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5, p6, p7, p8 interface{}) string,
	collection1, collection2, collection3, collection4, collection5, collection6, collection7, collection8 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5, p6, p7, p8)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		collection6,
		collection7,
		collection8,
		emptyCollection)
}

// VerifyAllCombinationsFor9 is for combinations of 9.
func VerifyAllCombinationsFor9(
	t Failable,
	header string,
	transform func(a, b, c, d, e, f, g, h, i interface{}) string,
	collection1,
	collection2,
	collection3,
	collection4,
	collection5,
	collection6,
	collection7,
	collection8,
	collection9 interface{}) error {

	if len(header) != 0 {
		header = fmt.Sprintf("%s\n\n\n", header)
	}

	var mapped []string

	slice1 := reflect.ValueOf(collection1)
	slice2 := reflect.ValueOf(collection2)
	slice3 := reflect.ValueOf(collection3)
	slice4 := reflect.ValueOf(collection4)
	slice5 := reflect.ValueOf(collection5)
	slice6 := reflect.ValueOf(collection6)
	slice7 := reflect.ValueOf(collection7)
	slice8 := reflect.ValueOf(collection8)
	slice9 := reflect.ValueOf(collection9)

	for i1 := 0; i1 < slice1.Len(); i1++ {
		for i2 := 0; i2 < slice2.Len(); i2++ {
			for i3 := 0; i3 < slice3.Len(); i3++ {
				for i4 := 0; i4 < slice4.Len(); i4++ {
					for i5 := 0; i5 < slice5.Len(); i5++ {
						for i6 := 0; i6 < slice6.Len(); i6++ {
							for i7 := 0; i7 < slice7.Len(); i7++ {
								for i8 := 0; i8 < slice8.Len(); i8++ {
									for i9 := 0; i9 < slice9.Len(); i9++ {
										p1 := slice1.Index(i1).Interface()
										p2 := slice2.Index(i2).Interface()
										p3 := slice3.Index(i3).Interface()
										p4 := slice4.Index(i4).Interface()
										p5 := slice5.Index(i5).Interface()
										p6 := slice6.Index(i6).Interface()
										p7 := slice7.Index(i7).Interface()
										p8 := slice8.Index(i8).Interface()
										p9 := slice9.Index(i9).Interface()

										parameterText := getParameterText(p1, p2, p3, p4, p5, p6, p7, p8, p9)
										transformText := getTransformText(transform, p1, p2, p3, p4, p5, p6, p7, p8, p9)
										if transformText != SkipThisCombination {
											mapped = append(mapped, fmt.Sprintf("%s => %s", parameterText, transformText))
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}

	outputText := header + strings.Join(mapped, "\n")
	return VerifyString(t, outputText)
}

func getParameterText(args ...interface{}) string {
	parameterText := "["
	for _, x := range args {
		if x != empty {
			parameterText += fmt.Sprintf("%v,", x)
		}
	}

	parameterText = parameterText[0 : len(parameterText)-1]
	parameterText += "]"

	return parameterText
}

func getTransformText(
	transform func(a, b, c, d, e, f, g, h, i interface{}) string,
	p1,
	p2,
	p3,
	p4,
	p5,
	p6,
	p7,
	p8,
	p9 interface{}) (s string) {
	defer func() {
		r := recover()
		if r != nil {
			s = "panic occurred"
		}
	}()

	return transform(p1, p2, p3, p4, p5, p6, p7, p8, p9)
}
<commit_msg>made exported variable a constant<commit_after>package approvaltests

import (
	"fmt"
	"strings"

	"reflect"
)

type emptyType struct{}

// SkipThisCombination should be returned if you do not want to process a particular combination.
const SkipThisCombination = "♬ SKIP THIS COMBINATION ♬"

var (
	empty           = emptyType{}
	emptyCollection = []emptyType{empty}
)

// VerifyAllCombinationsFor1 Example:
//   VerifyAllCombinationsFor1(t, "uppercase", func(x interface{}) string { return strings.ToUpper(x.(string)) }, []string("dog", "cat"})
func VerifyAllCombinationsFor1(t Failable, header string, transform func(interface{}) string, collection1 interface{}) error {
	transform2 := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1)
	}

	return VerifyAllCombinationsFor9(t, header, transform2, collection1,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor2 calls the transform function with all combinations
// from collection 1 and collection 2. The resulting received file contains all
// inputs and the resulting outputs. The received file is then compared to the
// approved version. If the transform function returns SkipThisCombination the
// output of this combination won't be displayed inside the received file.
func VerifyAllCombinationsFor2(
	t Failable,
	header string,
	transform func(p1, p2 interface{}) string,
	collection1, collection2 interface{}) error {
	transform2 := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2)
	}

	return VerifyAllCombinationsFor9(t, header, transform2, collection1,
		collection2,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor3 is for combinations of 3.
func VerifyAllCombinationsFor3(
	t Failable,
	header string,
	transform func(p1, p2, p3 interface{}) string,
	collection1, collection2, collection3 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor4 is for combinations of 4.
func VerifyAllCombinationsFor4(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4 interface{}) string,
	collection1, collection2, collection3, collection4 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor5 is for combinations of 5.
func VerifyAllCombinationsFor5(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5 interface{}) string,
	collection1, collection2, collection3, collection4, collection5 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		emptyCollection,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor6 is for combinations of 6.
func VerifyAllCombinationsFor6(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5, p6 interface{}) string,
	collection1, collection2, collection3, collection4, collection5, collection6 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5, p6)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		collection6,
		emptyCollection,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor7 is for combinations of 7.
func VerifyAllCombinationsFor7(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5, p6, p7 interface{}) string,
	collection1, collection2, collection3, collection4, collection5, collection6, collection7 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5, p6, p7)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		collection6,
		collection7,
		emptyCollection,
		emptyCollection)
}

// VerifyAllCombinationsFor8 is for combinations of 8.
func VerifyAllCombinationsFor8(
	t Failable,
	header string,
	transform func(p1, p2, p3, p4, p5, p6, p7, p8 interface{}) string,
	collection1, collection2, collection3, collection4, collection5, collection6, collection7, collection8 interface{}) error {

	kerning := func(p1, p2, p3, p4, p5, p6, p7, p8, p9 interface{}) string {
		return transform(p1, p2, p3, p4, p5, p6, p7, p8)
	}

	return VerifyAllCombinationsFor9(t, header, kerning,
		collection1,
		collection2,
		collection3,
		collection4,
		collection5,
		collection6,
		collection7,
		collection8,
		emptyCollection)
}

// VerifyAllCombinationsFor9 is for combinations of 9.
func VerifyAllCombinationsFor9( // nolint: funlen
	t Failable,
	header string,
	transform func(a, b, c, d, e, f, g, h, i interface{}) string,
	collection1,
	collection2,
	collection3,
	collection4,
	collection5,
	collection6,
	collection7,
	collection8,
	collection9 interface{}) error {

	if len(header) != 0 {
		header = fmt.Sprintf("%s\n\n\n", header)
	}

	var mapped []string

	slice1 := reflect.ValueOf(collection1)
	slice2 := reflect.ValueOf(collection2)
	slice3 := reflect.ValueOf(collection3)
	slice4 := reflect.ValueOf(collection4)
	slice5 := reflect.ValueOf(collection5)
	slice6 := reflect.ValueOf(collection6)
	slice7 := reflect.ValueOf(collection7)
	slice8 := reflect.ValueOf(collection8)
	slice9 := reflect.ValueOf(collection9)

	for i1 := 0; i1 < slice1.Len(); i1++ {
		for i2 := 0; i2 < slice2.Len(); i2++ {
			for i3 := 0; i3 < slice3.Len(); i3++ {
				for i4 := 0; i4 < slice4.Len(); i4++ {
					for i5 := 0; i5 < slice5.Len(); i5++ {
						for i6 := 0; i6 < slice6.Len(); i6++ {
							for i7 := 0; i7 < slice7.Len(); i7++ {
								for i8 := 0; i8 < slice8.Len(); i8++ {
									for i9 := 0; i9 < slice9.Len(); i9++ {
										p1 := slice1.Index(i1).Interface()
										p2 := slice2.Index(i2).Interface()
										p3 := slice3.Index(i3).Interface()
										p4 := slice4.Index(i4).Interface()
										p5 := slice5.Index(i5).Interface()
										p6 := slice6.Index(i6).Interface()
										p7 := slice7.Index(i7).Interface()
										p8 := slice8.Index(i8).Interface()
										p9 := slice9.Index(i9).Interface()

										parameterText := getParameterText(p1, p2, p3, p4, p5, p6, p7, p8, p9)
										transformText := getTransformText(transform, p1, p2, p3, p4, p5, p6, p7, p8, p9)
										if transformText != SkipThisCombination {
											mapped = append(mapped, fmt.Sprintf("%s => %s", parameterText, transformText))
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}

	outputText := header + strings.Join(mapped, "\n")
	return VerifyString(t, outputText)
}

func getParameterText(args ...interface{}) string {
	parameterText := "["
	for _, x := range args {
		if x != empty {
			parameterText += fmt.Sprintf("%v,", x)
		}
	}

	parameterText = parameterText[0 : len(parameterText)-1]
	parameterText += "]"

	return parameterText
}

func getTransformText(
	transform func(a, b, c, d, e, f, g, h, i interface{}) string,
	p1,
	p2,
	p3,
	p4,
	p5,
	p6,
	p7,
	p8,
	p9 interface{}) (s string) {
	defer func() {
		r := recover()
		if r != nil {
			s = "panic occurred"
		}
	}()

	return transform(p1, p2, p3, p4, p5, p6, p7, p8, p9)
}
<|endoftext|>


<commit_before>package commands

import (
	"io/ioutil"
	"os"
	"strings"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	pushCmd = &cobra.Command{
		Use:   "push",
		Short: "Push files to the Git LFS server",
		Run:   pushCommand,
	}
	pushDryRun       = false
	pushDeleteBranch = "(delete)"
	pushObjectIDs    = false
	useStdin         = false

	// shares some global vars and functions with commmands_pre_push.go
)

func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
	// Just use scanner here
	pointers, err := lfs.ScanRefs(left, right, nil)
	if err != nil {
		Panic(err, "Error scanning for Git LFS files")
	}

	totalSize := int64(0)
	for _, p := range pointers {
		totalSize += p.Size
	}

	uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(pointers), totalSize)

	for i, pointer := range pointers {
		if pushDryRun {
			Print("push %s", pointer.Name)
			continue
		}

		tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))

		u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
		if wErr != nil {
			if Debugging || wErr.Panic {
				Panic(wErr.Err, wErr.Error())
			} else {
				Exit(wErr.Error())
			}
		}
		uploadQueue.Add(u)
	}

	return uploadQueue
}

func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
	uploads := []*lfs.Uploadable{}
	totalSize := int64(0)

	for i, oid := range oids {
		if pushDryRun {
			Print("push object ID %s", oid)
			continue
		}
		tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))

		u, wErr := lfs.NewUploadable(oid, "")
		if wErr != nil {
			if Debugging || wErr.Panic {
				Panic(wErr.Err, wErr.Error())
			} else {
				Exit(wErr.Error())
			}
		}
		uploads = append(uploads, u)
	}

	uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(oids), totalSize)

	for _, u := range uploads {
		uploadQueue.Add(u)
	}

	return uploadQueue
}

// pushCommand pushes local objects to a Git LFS server.  It takes two
// arguments:
//
//   `<remote> <remote ref>`
//
// Both a remote name ("origin") or a remote URL are accepted.
//
// pushCommand calculates the git objects to send by looking comparing the range
// of commits between the local and remote git servers.
func pushCommand(cmd *cobra.Command, args []string) {
	var left, right string
	var uploadQueue *lfs.TransferQueue

	if len(args) == 0 {
		Print("Specify a remote and a remote branch name (`git lfs push origin master`)")
		os.Exit(1)
	}

	lfs.Config.CurrentRemote = args[0]

	if useStdin {
		requireStdin("Run this command from the Git pre-push hook, or leave the --stdin flag off.")

		// called from a pre-push hook!  Update the existing pre-push hook if it's
		// one that git-lfs set.
		lfs.InstallHooks(false)

		refsData, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			Panic(err, "Error reading refs on stdin")
		}

		if len(refsData) == 0 {
			return
		}

		left, right = decodeRefs(string(refsData))
		if left == pushDeleteBranch {
			return
		}

		uploadQueue = uploadsBetweenRefs(left, right)
	} else if pushObjectIDs {
		if len(args) < 2 {
			Print("Usage: git lfs push --object-id <remote> <lfs-object-id> [lfs-object-id] ...")
			return
		}

		uploadQueue = uploadsWithObjectIDs(args[1:])
	} else {
		var remoteArg, refArg string

		if len(args) < 1 {
			Print("Usage: git lfs push --dry-run <remote> [ref]")
			return
		}

		remoteArg = args[0]
		if len(args) == 2 {
			refArg = args[1]
		}

		localRef, err := git.CurrentRef()
		if err != nil {
			Panic(err, "Error getting local ref")
		}
		left = localRef

		remoteRef, err := git.LsRemote(remoteArg, refArg)
		if err != nil {
			Panic(err, "Error getting remote ref")
		}

		if remoteRef != "" {
			right = "^" + strings.Split(remoteRef, "\t")[0]
		}

		uploadQueue = uploadsBetweenRefs(left, right)
	}

	if !pushDryRun {
		uploadQueue.Wait()
		for _, err := range uploadQueue.Errors() {
			if Debugging || err.Panic {
				LoggedError(err.Err, err.Error())
			} else {
				Error(err.Error())
			}
		}

		if len(uploadQueue.Errors()) > 0 {
			os.Exit(2)
		}
	}
}

func init() {
	pushCmd.Flags().BoolVarP(&pushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
	pushCmd.Flags().BoolVarP(&useStdin, "stdin", "s", false, "Take refs on stdin (for pre-push hook)")
	pushCmd.Flags().BoolVarP(&pushObjectIDs, "object-id", "o", false, "Push LFS object ID(s)")
	RootCmd.AddCommand(pushCmd)
}
<commit_msg>Suppress progress output for a push dry run<commit_after>package commands

import (
	"io/ioutil"
	"os"
	"strings"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	pushCmd = &cobra.Command{
		Use:   "push",
		Short: "Push files to the Git LFS server",
		Run:   pushCommand,
	}
	pushDryRun       = false
	pushDeleteBranch = "(delete)"
	pushObjectIDs    = false
	useStdin         = false

	// shares some global vars and functions with commmands_pre_push.go
)

func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
	// Just use scanner here
	pointers, err := lfs.ScanRefs(left, right, nil)
	if err != nil {
		Panic(err, "Error scanning for Git LFS files")
	}

	totalSize := int64(0)
	for _, p := range pointers {
		totalSize += p.Size
	}

	uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(pointers), totalSize)

	for i, pointer := range pointers {
		if pushDryRun {
			Print("push %s", pointer.Name)
			continue
		}

		tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))

		u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
		if wErr != nil {
			if Debugging || wErr.Panic {
				Panic(wErr.Err, wErr.Error())
			} else {
				Exit(wErr.Error())
			}
		}
		uploadQueue.Add(u)
	}

	return uploadQueue
}

func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
	uploads := []*lfs.Uploadable{}
	totalSize := int64(0)

	for i, oid := range oids {
		if pushDryRun {
			Print("push object ID %s", oid)
			continue
		}
		tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))

		u, wErr := lfs.NewUploadable(oid, "")
		if wErr != nil {
			if Debugging || wErr.Panic {
				Panic(wErr.Err, wErr.Error())
			} else {
				Exit(wErr.Error())
			}
		}
		uploads = append(uploads, u)
	}

	uploadQueue := lfs.NewUploadQueue(lfs.Config.ConcurrentTransfers(), len(oids), totalSize)

	for _, u := range uploads {
		uploadQueue.Add(u)
	}

	return uploadQueue
}

// pushCommand pushes local objects to a Git LFS server.  It takes two
// arguments:
//
//   `<remote> <remote ref>`
//
// Both a remote name ("origin") or a remote URL are accepted.
//
// pushCommand calculates the git objects to send by looking comparing the range
// of commits between the local and remote git servers.
func pushCommand(cmd *cobra.Command, args []string) {
	var left, right string
	var uploadQueue *lfs.TransferQueue

	if len(args) == 0 {
		Print("Specify a remote and a remote branch name (`git lfs push origin master`)")
		os.Exit(1)
	}

	lfs.Config.CurrentRemote = args[0]

	if useStdin {
		requireStdin("Run this command from the Git pre-push hook, or leave the --stdin flag off.")

		// called from a pre-push hook!  Update the existing pre-push hook if it's
		// one that git-lfs set.
		lfs.InstallHooks(false)

		refsData, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			Panic(err, "Error reading refs on stdin")
		}

		if len(refsData) == 0 {
			return
		}

		left, right = decodeRefs(string(refsData))
		if left == pushDeleteBranch {
			return
		}

		uploadQueue = uploadsBetweenRefs(left, right)
	} else if pushObjectIDs {
		if len(args) < 2 {
			Print("Usage: git lfs push --object-id <remote> <lfs-object-id> [lfs-object-id] ...")
			return
		}

		uploadQueue = uploadsWithObjectIDs(args[1:])
	} else {
		var remoteArg, refArg string

		if len(args) < 1 {
			Print("Usage: git lfs push --dry-run <remote> [ref]")
			return
		}

		remoteArg = args[0]
		if len(args) == 2 {
			refArg = args[1]
		}

		localRef, err := git.CurrentRef()
		if err != nil {
			Panic(err, "Error getting local ref")
		}
		left = localRef

		remoteRef, err := git.LsRemote(remoteArg, refArg)
		if err != nil {
			Panic(err, "Error getting remote ref")
		}

		if remoteRef != "" {
			right = "^" + strings.Split(remoteRef, "\t")[0]
		}

		uploadQueue = uploadsBetweenRefs(left, right)
	}

	if !pushDryRun {
		uploadQueue.Wait()
		for _, err := range uploadQueue.Errors() {
			if Debugging || err.Panic {
				LoggedError(err.Err, err.Error())
			} else {
				Error(err.Error())
			}
		}

		if len(uploadQueue.Errors()) > 0 {
			os.Exit(2)
		}
	} else {
		uploadQueue.SuppressProgress()
	}
}

func init() {
	pushCmd.Flags().BoolVarP(&pushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
	pushCmd.Flags().BoolVarP(&useStdin, "stdin", "s", false, "Take refs on stdin (for pre-push hook)")
	pushCmd.Flags().BoolVarP(&pushObjectIDs, "object-id", "o", false, "Push LFS object ID(s)")
	RootCmd.AddCommand(pushCmd)
}
<|endoftext|>


<commit_before>package weed_server

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
	collectionName := r.FormValue("collection")
	collection, ok := ms.Topo.FindCollection(collectionName)
	if !ok {
		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("collection %s does not exist", collectionName))
		return
	}
	for _, server := range collection.ListVolumeServers() {
		err := operation.WithVolumeServerClient(server.Url(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
			_, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{
				Collection: collection.Name,
			})
			return deleteErr
		})
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		}
	}
	ms.Topo.DeleteCollection(collectionName)

	w.WriteHeader(http.StatusNoContent)
	return
}

func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Topology"] = ms.Topo.ToMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
	gcString := r.FormValue("garbageThreshold")
	gcThreshold := ms.option.GarbageThreshold
	if gcString != "" {
		var err error
		gcThreshold, err = strconv.ParseFloat(gcString, 32)
		if err != nil {
			glog.V(0).Infof("garbageThreshold %s is not a valid float number: %v", gcString, err)
			writeJsonError(w, r, http.StatusNotAcceptable, fmt.Errorf("garbageThreshold %s is not a valid float number", gcString))
			return
		}
	}
	// glog.Infoln("garbageThreshold =", gcThreshold)
	ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
	ms.dirStatusHandler(w, r)
}

func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
	count := 0
	option, err := ms.getVolumeGrowOption(r)
	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
		return
	}

	if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
		if ms.Topo.FreeSpace() < int64(count*option.ReplicaPlacement.GetCopyCount()) {
			err = fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.FreeSpace(), count*option.ReplicaPlacement.GetCopyCount())
		} else {
			count, err = ms.vg.GrowByCountAndType(ms.grpcDialOption, count, option, ms.Topo)
		}
	} else {
		err = fmt.Errorf("can not parse parameter count %s", r.FormValue("count"))
	}

	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
	} else {
		writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"count": count})
	}
}

func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Volumes"] = ms.Topo.ToVolumeMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) {
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	collection := r.FormValue("collection")
	location := ms.findVolumeLocation(collection, vid)
	if location.Error == "" {
		loc := location.Locations[rand.Intn(len(location.Locations))]
		var url string
		if r.URL.RawQuery != "" {
			url = util.NormalizeUrl(loc.PublicUrl) + r.URL.Path + "?" + r.URL.RawQuery
		} else {
			url = util.NormalizeUrl(loc.PublicUrl) + r.URL.Path
		}
		http.Redirect(w, r, url, http.StatusMovedPermanently)
	} else {
		writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %s not found: %s", vid, location.Error))
	}
}

func (ms *MasterServer) selfUrl(r *http.Request) string {
	if r.Host != "" {
		return r.Host
	}
	return "localhost:" + strconv.Itoa(ms.option.Port)
}
func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
	if ms.Topo.IsLeader() {
		submitForClientHandler(w, r, ms.selfUrl(r), ms.grpcDialOption)
	} else {
		masterUrl, err := ms.Topo.Leader()
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
		} else {
			submitForClientHandler(w, r, masterUrl, ms.grpcDialOption)
		}
	}
}

func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
	return vl.GetActiveVolumeCount(option) > 0
}

func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGrowOption, error) {
	replicationString := r.FormValue("replication")
	if replicationString == "" {
		replicationString = ms.option.DefaultReplicaPlacement
	}
	replicaPlacement, err := super_block.NewReplicaPlacementFromString(replicationString)
	if err != nil {
		return nil, err
	}
	ttl, err := needle.ReadTTL(r.FormValue("ttl"))
	if err != nil {
		return nil, err
	}
	memoryMapMaxSizeMb, err := memory_map.ReadMemoryMapMaxSizeMb(r.FormValue("memoryMapMaxSizeMb"))
	if err != nil {
		return nil, err
	}

	preallocate := ms.preallocateSize
	if r.FormValue("preallocate") != "" {
		preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64)
		if err != nil {
			return nil, fmt.Errorf("Failed to parse int64 preallocate = %s: %v", r.FormValue("preallocate"), err)
		}
	}
	volumeGrowOption := &topology.VolumeGrowOption{
		Collection:         r.FormValue("collection"),
		ReplicaPlacement:   replicaPlacement,
		Ttl:                ttl,
		Prealloacte:        preallocate,
		DataCenter:         r.FormValue("dataCenter"),
		Rack:               r.FormValue("rack"),
		DataNode:           r.FormValue("dataNode"),
		MemoryMapMaxSizeMb: memoryMapMaxSizeMb,
	}
	return volumeGrowOption, nil
}
<commit_msg>master:  changing 301 to 308 on redirect to volume servers<commit_after>package weed_server

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
	collectionName := r.FormValue("collection")
	collection, ok := ms.Topo.FindCollection(collectionName)
	if !ok {
		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("collection %s does not exist", collectionName))
		return
	}
	for _, server := range collection.ListVolumeServers() {
		err := operation.WithVolumeServerClient(server.Url(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
			_, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{
				Collection: collection.Name,
			})
			return deleteErr
		})
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		}
	}
	ms.Topo.DeleteCollection(collectionName)

	w.WriteHeader(http.StatusNoContent)
	return
}

func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Topology"] = ms.Topo.ToMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
	gcString := r.FormValue("garbageThreshold")
	gcThreshold := ms.option.GarbageThreshold
	if gcString != "" {
		var err error
		gcThreshold, err = strconv.ParseFloat(gcString, 32)
		if err != nil {
			glog.V(0).Infof("garbageThreshold %s is not a valid float number: %v", gcString, err)
			writeJsonError(w, r, http.StatusNotAcceptable, fmt.Errorf("garbageThreshold %s is not a valid float number", gcString))
			return
		}
	}
	// glog.Infoln("garbageThreshold =", gcThreshold)
	ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
	ms.dirStatusHandler(w, r)
}

func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
	count := 0
	option, err := ms.getVolumeGrowOption(r)
	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
		return
	}

	if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
		if ms.Topo.FreeSpace() < int64(count*option.ReplicaPlacement.GetCopyCount()) {
			err = fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.FreeSpace(), count*option.ReplicaPlacement.GetCopyCount())
		} else {
			count, err = ms.vg.GrowByCountAndType(ms.grpcDialOption, count, option, ms.Topo)
		}
	} else {
		err = fmt.Errorf("can not parse parameter count %s", r.FormValue("count"))
	}

	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
	} else {
		writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"count": count})
	}
}

func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Volumes"] = ms.Topo.ToVolumeMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) {
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	collection := r.FormValue("collection")
	location := ms.findVolumeLocation(collection, vid)
	if location.Error == "" {
		loc := location.Locations[rand.Intn(len(location.Locations))]
		var url string
		if r.URL.RawQuery != "" {
			url = util.NormalizeUrl(loc.PublicUrl) + r.URL.Path + "?" + r.URL.RawQuery
		} else {
			url = util.NormalizeUrl(loc.PublicUrl) + r.URL.Path
		}
		http.Redirect(w, r, url, http.StatusPermanentRedirect)
	} else {
		writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %s not found: %s", vid, location.Error))
	}
}

func (ms *MasterServer) selfUrl(r *http.Request) string {
	if r.Host != "" {
		return r.Host
	}
	return "localhost:" + strconv.Itoa(ms.option.Port)
}
func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
	if ms.Topo.IsLeader() {
		submitForClientHandler(w, r, ms.selfUrl(r), ms.grpcDialOption)
	} else {
		masterUrl, err := ms.Topo.Leader()
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
		} else {
			submitForClientHandler(w, r, masterUrl, ms.grpcDialOption)
		}
	}
}

func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
	return vl.GetActiveVolumeCount(option) > 0
}

func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGrowOption, error) {
	replicationString := r.FormValue("replication")
	if replicationString == "" {
		replicationString = ms.option.DefaultReplicaPlacement
	}
	replicaPlacement, err := super_block.NewReplicaPlacementFromString(replicationString)
	if err != nil {
		return nil, err
	}
	ttl, err := needle.ReadTTL(r.FormValue("ttl"))
	if err != nil {
		return nil, err
	}
	memoryMapMaxSizeMb, err := memory_map.ReadMemoryMapMaxSizeMb(r.FormValue("memoryMapMaxSizeMb"))
	if err != nil {
		return nil, err
	}

	preallocate := ms.preallocateSize
	if r.FormValue("preallocate") != "" {
		preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64)
		if err != nil {
			return nil, fmt.Errorf("Failed to parse int64 preallocate = %s: %v", r.FormValue("preallocate"), err)
		}
	}
	volumeGrowOption := &topology.VolumeGrowOption{
		Collection:         r.FormValue("collection"),
		ReplicaPlacement:   replicaPlacement,
		Ttl:                ttl,
		Prealloacte:        preallocate,
		DataCenter:         r.FormValue("dataCenter"),
		Rack:               r.FormValue("rack"),
		DataNode:           r.FormValue("dataNode"),
		MemoryMapMaxSizeMb: memoryMapMaxSizeMb,
	}
	return volumeGrowOption, nil
}
<|endoftext|>


<commit_before>package needle_map

import (
	"fmt"
	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"testing"
)

func TestOverflow2(t *testing.T) {
	m := NewCompactMap()
	_, oldSize := m.Set(NeedleId(150088), ToOffset(8), 3000073)
	if oldSize != 0 {
		t.Fatalf("expecting no previous data")
	}
	_, oldSize = m.Set(NeedleId(150088), ToOffset(8), 3000073)
	if oldSize != 3000073 {
		t.Fatalf("expecting previous data size is %d, not %d", 3000073, oldSize)
	}
	m.Set(NeedleId(150073), ToOffset(8), 3000073)
	m.Set(NeedleId(150089), ToOffset(8), 3000073)
	m.Set(NeedleId(150076), ToOffset(8), 3000073)
	m.Set(NeedleId(150124), ToOffset(8), 3000073)
	m.Set(NeedleId(150137), ToOffset(8), 3000073)
	m.Set(NeedleId(150147), ToOffset(8), 3000073)
	m.Set(NeedleId(150145), ToOffset(8), 3000073)
	m.Set(NeedleId(150158), ToOffset(8), 3000073)
	m.Set(NeedleId(150162), ToOffset(8), 3000073)

	m.AscendingVisit(func(value NeedleValue) error {
		println("needle key:", value.Key)
		return nil
	})
}

func TestIssue52(t *testing.T) {
	m := NewCompactMap()
	m.Set(NeedleId(10002), ToOffset(10002), 10002)
	if element, ok := m.Get(NeedleId(10002)); ok {
		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
	}
	m.Set(NeedleId(10001), ToOffset(10001), 10001)
	if element, ok := m.Get(NeedleId(10002)); ok {
		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
	} else {
		t.Fatal("key 10002 missing after setting 10001")
	}
}

func TestCompactMap(t *testing.T) {
	m := NewCompactMap()
	for i := uint32(0); i < 100*batch; i += 2 {
		m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
	}

	for i := uint32(0); i < 100*batch; i += 37 {
		m.Delete(NeedleId(i))
	}

	for i := uint32(0); i < 10*batch; i += 3 {
		m.Set(NeedleId(i), ToOffset(int64(i+11)), Size(i+5))
	}

	//	for i := uint32(0); i < 100; i++ {
	//		if v := m.Get(Key(i)); v != nil {
	//			glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
	//		}
	//	}

	for i := uint32(0); i < 10*batch; i++ {
		v, ok := m.Get(NeedleId(i))
		if i%3 == 0 {
			if !ok {
				t.Fatal("key", i, "missing!")
			}
			if v.Size != Size(i+5) {
				t.Fatal("key", i, "size", v.Size)
			}
		} else if i%37 == 0 {
			if ok && v.Size.IsValid() {
				t.Fatal("key", i, "should have been deleted needle value", v)
			}
		} else if i%2 == 0 {
			if v.Size != Size(i) {
				t.Fatal("key", i, "size", v.Size)
			}
		}
	}

	for i := uint32(10 * batch); i < 100*batch; i++ {
		v, ok := m.Get(NeedleId(i))
		if i%37 == 0 {
			if ok && v.Size.IsValid() {
				t.Fatal("key", i, "should have been deleted needle value", v)
			}
		} else if i%2 == 0 {
			if v == nil {
				t.Fatal("key", i, "missing")
			}
			if v.Size != Size(i) {
				t.Fatal("key", i, "size", v.Size)
			}
		}
	}

}

func TestOverflow(t *testing.T) {
	cs := NewCompactSection(1)

	cs.setOverflowEntry(1, ToOffset(12), 12)
	cs.setOverflowEntry(2, ToOffset(12), 12)
	cs.setOverflowEntry(3, ToOffset(12), 12)
	cs.setOverflowEntry(4, ToOffset(12), 12)
	cs.setOverflowEntry(5, ToOffset(12), 12)

	if cs.overflow[2].Key != 3 {
		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
	}

	cs.setOverflowEntry(3, ToOffset(24), 24)

	if cs.overflow[2].Key != 3 {
		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
	}

	if cs.overflow[2].Size != 24 {
		t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
	}

	cs.deleteOverflowEntry(4)

	if len(cs.overflow) != 5 {
		t.Fatalf("expecting 5 entries now: %+v", cs.overflow)
	}

	_, x, _ := cs.findOverflowEntry(5)
	if x.Key != 5 {
		t.Fatalf("expecting entry 5 now: %+v", x)
	}

	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key)
	}
	println()

	cs.deleteOverflowEntry(1)

	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key, "size", x.Size)
	}
	println()

	cs.setOverflowEntry(4, ToOffset(44), 44)
	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key)
	}
	println()

	cs.setOverflowEntry(1, ToOffset(11), 11)

	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key)
	}
	println()

}
<commit_msg>test compact map with snowflake sequencer<commit_after>package needle_map

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/sequence"
	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"testing"
)

func TestSnowflakeSequencer(t *testing.T) {
	m := NewCompactMap()
	seq, _ := sequence.NewSnowflakeSequencer("for_test", 1)

	for i := 0; i < 200000; i++ {
		id := seq.NextFileId(1)
		oldOffset, oldSize := m.Set(NeedleId(id), ToOffset(8), 3000073)
		if oldSize != 0 {
			t.Errorf("id %d oldOffset %v oldSize %d", id, oldOffset, oldSize)
		}
	}

}

func TestOverflow2(t *testing.T) {
	m := NewCompactMap()
	_, oldSize := m.Set(NeedleId(150088), ToOffset(8), 3000073)
	if oldSize != 0 {
		t.Fatalf("expecting no previous data")
	}
	_, oldSize = m.Set(NeedleId(150088), ToOffset(8), 3000073)
	if oldSize != 3000073 {
		t.Fatalf("expecting previous data size is %d, not %d", 3000073, oldSize)
	}
	m.Set(NeedleId(150073), ToOffset(8), 3000073)
	m.Set(NeedleId(150089), ToOffset(8), 3000073)
	m.Set(NeedleId(150076), ToOffset(8), 3000073)
	m.Set(NeedleId(150124), ToOffset(8), 3000073)
	m.Set(NeedleId(150137), ToOffset(8), 3000073)
	m.Set(NeedleId(150147), ToOffset(8), 3000073)
	m.Set(NeedleId(150145), ToOffset(8), 3000073)
	m.Set(NeedleId(150158), ToOffset(8), 3000073)
	m.Set(NeedleId(150162), ToOffset(8), 3000073)

	m.AscendingVisit(func(value NeedleValue) error {
		println("needle key:", value.Key)
		return nil
	})
}

func TestIssue52(t *testing.T) {
	m := NewCompactMap()
	m.Set(NeedleId(10002), ToOffset(10002), 10002)
	if element, ok := m.Get(NeedleId(10002)); ok {
		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
	}
	m.Set(NeedleId(10001), ToOffset(10001), 10001)
	if element, ok := m.Get(NeedleId(10002)); ok {
		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
	} else {
		t.Fatal("key 10002 missing after setting 10001")
	}
}

func TestCompactMap(t *testing.T) {
	m := NewCompactMap()
	for i := uint32(0); i < 100*batch; i += 2 {
		m.Set(NeedleId(i), ToOffset(int64(i)), Size(i))
	}

	for i := uint32(0); i < 100*batch; i += 37 {
		m.Delete(NeedleId(i))
	}

	for i := uint32(0); i < 10*batch; i += 3 {
		m.Set(NeedleId(i), ToOffset(int64(i+11)), Size(i+5))
	}

	//	for i := uint32(0); i < 100; i++ {
	//		if v := m.Get(Key(i)); v != nil {
	//			glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
	//		}
	//	}

	for i := uint32(0); i < 10*batch; i++ {
		v, ok := m.Get(NeedleId(i))
		if i%3 == 0 {
			if !ok {
				t.Fatal("key", i, "missing!")
			}
			if v.Size != Size(i+5) {
				t.Fatal("key", i, "size", v.Size)
			}
		} else if i%37 == 0 {
			if ok && v.Size.IsValid() {
				t.Fatal("key", i, "should have been deleted needle value", v)
			}
		} else if i%2 == 0 {
			if v.Size != Size(i) {
				t.Fatal("key", i, "size", v.Size)
			}
		}
	}

	for i := uint32(10 * batch); i < 100*batch; i++ {
		v, ok := m.Get(NeedleId(i))
		if i%37 == 0 {
			if ok && v.Size.IsValid() {
				t.Fatal("key", i, "should have been deleted needle value", v)
			}
		} else if i%2 == 0 {
			if v == nil {
				t.Fatal("key", i, "missing")
			}
			if v.Size != Size(i) {
				t.Fatal("key", i, "size", v.Size)
			}
		}
	}

}

func TestOverflow(t *testing.T) {
	cs := NewCompactSection(1)

	cs.setOverflowEntry(1, ToOffset(12), 12)
	cs.setOverflowEntry(2, ToOffset(12), 12)
	cs.setOverflowEntry(3, ToOffset(12), 12)
	cs.setOverflowEntry(4, ToOffset(12), 12)
	cs.setOverflowEntry(5, ToOffset(12), 12)

	if cs.overflow[2].Key != 3 {
		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
	}

	cs.setOverflowEntry(3, ToOffset(24), 24)

	if cs.overflow[2].Key != 3 {
		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
	}

	if cs.overflow[2].Size != 24 {
		t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
	}

	cs.deleteOverflowEntry(4)

	if len(cs.overflow) != 5 {
		t.Fatalf("expecting 5 entries now: %+v", cs.overflow)
	}

	_, x, _ := cs.findOverflowEntry(5)
	if x.Key != 5 {
		t.Fatalf("expecting entry 5 now: %+v", x)
	}

	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key)
	}
	println()

	cs.deleteOverflowEntry(1)

	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key, "size", x.Size)
	}
	println()

	cs.setOverflowEntry(4, ToOffset(44), 44)
	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key)
	}
	println()

	cs.setOverflowEntry(1, ToOffset(11), 11)

	for i, x := range cs.overflow {
		println("overflow[", i, "]:", x.Key)
	}
	println()

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/google/go-github/github"
)

const (
	TestOwner = "tcnksm"
	TestRepo  = "github-api-test"
)

func testGithubClient(t *testing.T) GitHub {
	token := os.Getenv(EnvGitHubToken)
	client, err := NewGitHubClient(TestOwner, TestRepo, token, defaultBaseURL)
	if err != nil {
		t.Fatal("NewGitHubClient failed:", err)
	}
	return client
}

func TestGitHubClient(t *testing.T) {
	t.Parallel()

	c := testGithubClient(t)
	testTag := "github-client"
	cases := []struct {
		Request *github.RepositoryRelease
	}{
		{
			&github.RepositoryRelease{
				TagName:    github.String(testTag),
				Draft:      github.Bool(false),
				Prerelease: github.Bool(false),
			},
		},

		{
			&github.RepositoryRelease{
				TagName:    github.String(testTag),
				Draft:      github.Bool(false),
				Prerelease: github.Bool(true),
			},
		},

		{
			&github.RepositoryRelease{
				TagName:    github.String(testTag),
				Draft:      github.Bool(true),
				Prerelease: github.Bool(false),
			},
		},
	}

	for i, tc := range cases {
		// Prevent a lot of requests at same time to GitHub API
		time.Sleep(1 * time.Second)

		created, err := c.CreateRelease(context.TODO(), tc.Request)
		if err != nil {
			t.Fatalf("#%d CreateRelease failed: %s", i, err)
		}

		// Draft release doesn't create tag. So it's not found.
		if !*created.Draft {
			got, err := c.GetRelease(context.TODO(), *created.TagName)
			if err != nil {
				t.Fatalf("#%d GetRelease failed: %s", i, err)
			}

			if *got.ID != *created.ID {
				t.Fatalf("got ID = %d, want %d", *got.ID, *created.ID)
			}
		}

		if err := c.DeleteRelease(context.TODO(), *created.ID); err != nil {
			t.Fatalf("#%d DeleteRelease failed: %s", i, err)
		}

		if *created.Draft {
			continue
		}

		if err := c.DeleteTag(context.TODO(), *tc.Request.TagName); err != nil {
			t.Fatalf("#%d DeleteTag failed: %s", i, err)
		}
	}
}

func TestGitHubClient_Upload(t *testing.T) {
	client := testGithubClient(t)
	testTag := "github-client-upload-asset"
	req := &github.RepositoryRelease{
		TagName: github.String(testTag),
		Draft:   github.Bool(true),
	}

	release, err := client.CreateRelease(context.TODO(), req)
	if err != nil {
		t.Fatal("CreateRelease failed:", err)
	}

	defer func() {
		if err := client.DeleteRelease(context.TODO(), *release.ID); err != nil {
			t.Fatalf("DeleteRelease failed: %s", err)
		}
	}()

	filename := filepath.Join("./testdata", "darwin_386")
	asset, err := client.UploadAsset(context.TODO(), *release.ID, filename)
	if err != nil {
		t.Fatal("UploadAsset failed:", err)
	}

	githubClient, ok := client.(*GitHubClient)
	if !ok {
		t.Fatal("Faield to asset to GithubClient")
	}

	rc, url, err := githubClient.Repositories.DownloadReleaseAsset(
		githubClient.Owner, githubClient.Repo, *asset.ID)
	if err != nil {
		t.Fatal("DownloadReleaseAsset failed:", err)
	}

	var buf bytes.Buffer
	if len(url) != 0 {
		res, err := http.Get(url)
		if err != nil {
			t.Fatal("http.Get failed:", err)
		}

		if _, err := io.Copy(&buf, res.Body); err != nil {
			t.Fatal("Copy failed:", err)
		}
		res.Body.Close()

	} else {
		if _, err := io.Copy(&buf, rc); err != nil {
			t.Fatal("Copy failed:", err)
		}
		rc.Close()

	}

	if got, want := buf.String(), "darwin_386\n"; got != want {
		t.Fatalf("file body is %q, want %q", got, want)
	}
}

func TestGitHubClient_ListAssets(t *testing.T) {
	client := testGithubClient(t)
	testTag := "github-list-assets"
	req := &github.RepositoryRelease{
		TagName: github.String(testTag),
		Draft:   github.Bool(true),
	}

	release, err := client.CreateRelease(context.TODO(), req)
	if err != nil {
		t.Fatal("CreateRelease failed:", err)
	}

	defer func() {
		if err := client.DeleteRelease(context.TODO(), *release.ID); err != nil {
			t.Fatalf("DeleteRelease failed: %s", err)
		}
	}()

	for _, filename := range []string{"darwin_386", "darwin_amd64"} {
		filename := filepath.Join("./testdata", filename)
		if _, err := client.UploadAsset(context.TODO(), *release.ID, filename); err != nil {
			t.Fatal("UploadAsset failed:", err)
		}
	}

	assets, err := client.ListAssets(context.TODO(), *release.ID)
	if err != nil {
		t.Fatal("ListAssets failed:", err)
	}

	if got, want := len(assets), 2; got != want {
		t.Fatalf("ListAssets number = %d, want %d", got, want)
	}

	if err := client.DeleteAsset(context.TODO(), *assets[0].ID); err != nil {
		t.Fatal("DeleteAsset failed:", err)
	}

	assets, err = client.ListAssets(context.TODO(), *release.ID)
	if err != nil {
		t.Fatal("ListAssets failed:", err)
	}

	if got, want := len(assets), 1; got != want {
		t.Fatalf("ListAssets number = %d, want %d", got, want)
	}
}
<commit_msg>change test repository<commit_after>package main

import (
	"bytes"
	"context"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/google/go-github/github"
)

const (
	TestOwner = "ghtools"
	TestRepo  = "github-api-test"
)

func testGithubClient(t *testing.T) GitHub {
	token := os.Getenv(EnvGitHubToken)
	client, err := NewGitHubClient(TestOwner, TestRepo, token, defaultBaseURL)
	if err != nil {
		t.Fatal("NewGitHubClient failed:", err)
	}
	return client
}

func TestGitHubClient(t *testing.T) {
	t.Parallel()

	c := testGithubClient(t)
	testTag := "github-client"
	cases := []struct {
		Request *github.RepositoryRelease
	}{
		{
			&github.RepositoryRelease{
				TagName:    github.String(testTag),
				Draft:      github.Bool(false),
				Prerelease: github.Bool(false),
			},
		},

		{
			&github.RepositoryRelease{
				TagName:    github.String(testTag),
				Draft:      github.Bool(false),
				Prerelease: github.Bool(true),
			},
		},

		{
			&github.RepositoryRelease{
				TagName:    github.String(testTag),
				Draft:      github.Bool(true),
				Prerelease: github.Bool(false),
			},
		},
	}

	for i, tc := range cases {
		// Prevent a lot of requests at same time to GitHub API
		time.Sleep(1 * time.Second)

		created, err := c.CreateRelease(context.TODO(), tc.Request)
		if err != nil {
			t.Fatalf("#%d CreateRelease failed: %s", i, err)
		}

		// Draft release doesn't create tag. So it's not found.
		if !*created.Draft {
			got, err := c.GetRelease(context.TODO(), *created.TagName)
			if err != nil {
				t.Fatalf("#%d GetRelease failed: %s", i, err)
			}

			if *got.ID != *created.ID {
				t.Fatalf("got ID = %d, want %d", *got.ID, *created.ID)
			}
		}

		if err := c.DeleteRelease(context.TODO(), *created.ID); err != nil {
			t.Fatalf("#%d DeleteRelease failed: %s", i, err)
		}

		if *created.Draft {
			continue
		}

		if err := c.DeleteTag(context.TODO(), *tc.Request.TagName); err != nil {
			t.Fatalf("#%d DeleteTag failed: %s", i, err)
		}
	}
}

func TestGitHubClient_Upload(t *testing.T) {
	client := testGithubClient(t)
	testTag := "github-client-upload-asset"
	req := &github.RepositoryRelease{
		TagName: github.String(testTag),
		Draft:   github.Bool(true),
	}

	release, err := client.CreateRelease(context.TODO(), req)
	if err != nil {
		t.Fatal("CreateRelease failed:", err)
	}

	defer func() {
		if err := client.DeleteRelease(context.TODO(), *release.ID); err != nil {
			t.Fatalf("DeleteRelease failed: %s", err)
		}
	}()

	filename := filepath.Join("./testdata", "darwin_386")
	asset, err := client.UploadAsset(context.TODO(), *release.ID, filename)
	if err != nil {
		t.Fatal("UploadAsset failed:", err)
	}

	githubClient, ok := client.(*GitHubClient)
	if !ok {
		t.Fatal("Faield to asset to GithubClient")
	}

	rc, url, err := githubClient.Repositories.DownloadReleaseAsset(
		githubClient.Owner, githubClient.Repo, *asset.ID)
	if err != nil {
		t.Fatal("DownloadReleaseAsset failed:", err)
	}

	var buf bytes.Buffer
	if len(url) != 0 {
		res, err := http.Get(url)
		if err != nil {
			t.Fatal("http.Get failed:", err)
		}

		if _, err := io.Copy(&buf, res.Body); err != nil {
			t.Fatal("Copy failed:", err)
		}
		res.Body.Close()

	} else {
		if _, err := io.Copy(&buf, rc); err != nil {
			t.Fatal("Copy failed:", err)
		}
		rc.Close()

	}

	if got, want := buf.String(), "darwin_386\n"; got != want {
		t.Fatalf("file body is %q, want %q", got, want)
	}
}

func TestGitHubClient_ListAssets(t *testing.T) {
	client := testGithubClient(t)
	testTag := "github-list-assets"
	req := &github.RepositoryRelease{
		TagName: github.String(testTag),
		Draft:   github.Bool(true),
	}

	release, err := client.CreateRelease(context.TODO(), req)
	if err != nil {
		t.Fatal("CreateRelease failed:", err)
	}

	defer func() {
		if err := client.DeleteRelease(context.TODO(), *release.ID); err != nil {
			t.Fatalf("DeleteRelease failed: %s", err)
		}
	}()

	for _, filename := range []string{"darwin_386", "darwin_amd64"} {
		filename := filepath.Join("./testdata", filename)
		if _, err := client.UploadAsset(context.TODO(), *release.ID, filename); err != nil {
			t.Fatal("UploadAsset failed:", err)
		}
	}

	assets, err := client.ListAssets(context.TODO(), *release.ID)
	if err != nil {
		t.Fatal("ListAssets failed:", err)
	}

	if got, want := len(assets), 2; got != want {
		t.Fatalf("ListAssets number = %d, want %d", got, want)
	}

	if err := client.DeleteAsset(context.TODO(), *assets[0].ID); err != nil {
		t.Fatal("DeleteAsset failed:", err)
	}

	assets, err = client.ListAssets(context.TODO(), *release.ID)
	if err != nil {
		t.Fatal("ListAssets failed:", err)
	}

	if got, want := len(assets), 1; got != want {
		t.Fatalf("ListAssets number = %d, want %d", got, want)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"code.google.com/p/go.crypto/ssh"
	"github.com/flynn/go-shlex"
)

const PrereceiveHookTmpl = `#!/bin/bash
set -eo pipefail; while read oldrev newrev refname; do
[[ $refname = "refs/heads/master" ]] && git archive $newrev | {{RECEIVER}} "$RECEIVE_REPO" "$newrev" | sed -$([[ $(uname) == "Darwin" ]] && echo l || echo u) "s/^/"$'\e[1G'"/"
done
`

var prereceiveHook []byte

var port = flag.String("p", "22", "port to listen on")
var repoPath = flag.String("r", "/tmp/repos", "path to repo cache")
var noAuth = flag.Bool("n", false, "disable client authentication")
var keys = flag.String("k", "", "pem file containing private keys (read from SSH_PRIVATE_KEYS by default)")

var authChecker []string

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [options] <authchecker> <receiver>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	if flag.NArg() < 2 {
		flag.Usage()
		os.Exit(64)
	}
	var err error
	authChecker, err = shlex.Split(flag.Arg(0))
	if err != nil {
		log.Fatalln("Unable to parse authchecker command:", err)
	}

	receiver, err := shlex.Split(flag.Arg(1))
	if err != nil {
		log.Fatalln("Unable to parse receiver command:", err)
	}
	receiver[0], err = filepath.Abs(receiver[0])
	if err != nil {
		log.Fatalln("Invalid receiver path:", err)
	}
	prereceiveHook = []byte(strings.Replace(PrereceiveHookTmpl, "{{RECEIVER}}", strings.Join(receiver, " "), 1))

	var config *ssh.ServerConfig
	if *noAuth {
		config = &ssh.ServerConfig{NoClientAuth: true}
	} else {
		config = &ssh.ServerConfig{PublicKeyCallback: checkAuth}
	}

	if keyEnv := os.Getenv("SSH_PRIVATE_KEYS"); keyEnv != "" {
		if err := parseKeys(config, []byte(keyEnv)); err != nil {
			log.Fatalln("Failed to load private keys:", err)
		}
	} else {
		pemBytes, err := ioutil.ReadFile(*keys)
		if err != nil {
			log.Fatalln("Failed to load private keys:", err)
		}
		if err := parseKeys(config, pemBytes); err != nil {
			log.Fatalln("Failed to parse private keys:", err)
		}
	}

	if p := os.Getenv("PORT"); p != "" && *port == "22" {
		*port = p
	}
	listener, err := net.Listen("tcp", ":"+*port)
	if err != nil {
		log.Fatalln("Failed to listen for connections:", err)
	}
	for {
		// SSH connections just house multiplexed connections
		conn, err := listener.Accept()
		if err != nil {
			log.Println("Failed to accept incoming connection:", err)
			continue
		}
		go handleConn(conn, config)
	}
}

func parseKeys(conf *ssh.ServerConfig, pemData []byte) error {
	var found bool
	for {
		var block *pem.Block
		block, pemData = pem.Decode(pemData)
		if block == nil {
			if !found {
				return errors.New("no private keys found")
			}
			return nil
		}
		if err := addKey(conf, block); err != nil {
			return err
		}
		found = true
	}
}

func addKey(conf *ssh.ServerConfig, block *pem.Block) (err error) {
	var key interface{}
	switch block.Type {
	case "RSA PRIVATE KEY":
		key, err = x509.ParsePKCS1PrivateKey(block.Bytes)
	case "EC PRIVATE KEY":
		key, err = x509.ParseECPrivateKey(block.Bytes)
	case "DSA PRIVATE KEY":
		key, err = ssh.ParseDSAPrivateKey(block.Bytes)
	default:
		return fmt.Errorf("unsupported key type %q", block.Type)
	}
	if err != nil {
		return err
	}
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		return err
	}
	conf.AddHostKey(signer)
	return nil
}

func handleConn(conn net.Conn, conf *ssh.ServerConfig) {
	defer conn.Close()
	sshConn, chans, reqs, err := ssh.NewServerConn(conn, conf)
	if err != nil {
		log.Println("Failed to handshake:", err)
		return
	}

	go ssh.DiscardRequests(reqs)

	for ch := range chans {
		if ch.ChannelType() != "session" {
			ch.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		go handleChannel(sshConn, ch)
	}
}

func handleChannel(conn *ssh.ServerConn, newChan ssh.NewChannel) {
	ch, reqs, err := newChan.Accept()
	if err != nil {
		log.Println("newChan.Accept failed:", err)
		return
	}
	defer ch.Close()
	for req := range reqs {
		switch req.Type {
		case "exec":
			fail := func(at string, err error) {
				log.Printf("%s failed: %s", at, err)
				ch.Stderr().Write([]byte("Internal error.\n"))
			}
			if req.WantReply {
				req.Reply(true, nil)
			}
			cmdline := string(req.Payload[4:])
			cmdargs, err := shlex.Split(cmdline)
			if err != nil || len(cmdargs) != 2 {
				ch.Stderr().Write([]byte("Invalid arguments.\n"))
				return
			}
			if cmdargs[0] != "git-receive-pack" {
				ch.Stderr().Write([]byte("Only `git push` is supported.\n"))
				return
			}
			cmdargs[1] = strings.TrimSuffix(strings.TrimPrefix(cmdargs[1], "/"), ".git")
			if strings.Contains(cmdargs[1], "..") {
				ch.Stderr().Write([]byte("Invalid repo.\n"))
				return
			}

			if err := ensureCacheRepo(cmdargs[1]); err != nil {
				fail("ensureCacheRepo", err)
				return
			}
			cmd := exec.Command("git-shell", "-c", cmdargs[0]+" '"+cmdargs[1]+"'")
			cmd.Dir = *repoPath
			cmd.Env = append(os.Environ(),
				"RECEIVE_USER="+conn.User(),
				"RECEIVE_REPO="+cmdargs[1],
			)
			done, err := attachCmd(cmd, ch, ch.Stderr(), ch)
			if err != nil {
				fail("attachCmd", err)
				return
			}
			if err := cmd.Start(); err != nil {
				fail("cmd.Start", err)
				return
			}
			done.Wait()
			status, err := exitStatus(cmd.Wait())
			if err != nil {
				fail("exitStatus", err)
				return
			}
			if _, err := ch.SendRequest("exit-status", false, ssh.Marshal(&status)); err != nil {
				fail("sendExit", err)
			}
			return
		case "env":
			if req.WantReply {
				req.Reply(true, nil)
			}
		}
	}
}

var ErrUnauthorized = errors.New("gitreceive: user is unauthorized")

func checkAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
	status, err := exitStatus(exec.Command(authChecker[0],
		append(authChecker[1:], conn.User(), string(bytes.TrimSpace(ssh.MarshalAuthorizedKey(key))))...).Run())
	if err != nil {
		return nil, err
	}
	if status.Status == 0 {
		return nil, nil
	}
	return nil, ErrUnauthorized
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (*sync.WaitGroup, error) {
	var wg sync.WaitGroup
	wg.Add(2)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	go func() {
		io.Copy(stdinIn, stdin)
		stdinIn.Close()
	}()
	go func() {
		io.Copy(stdout, stdoutOut)
		wg.Done()
	}()
	go func() {
		io.Copy(stderr, stderrOut)
		wg.Done()
	}()

	return &wg, nil
}

type exitStatusMsg struct {
	Status uint32
}

func exitStatus(err error) (exitStatusMsg, error) {
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// There is no platform independent way to retrieve
			// the exit code, but the following will work on Unix
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return exitStatusMsg{uint32(status.ExitStatus())}, nil
			}
		}
		return exitStatusMsg{0}, err
	}
	return exitStatusMsg{0}, nil
}

var cacheMtx sync.Mutex

func ensureCacheRepo(path string) error {
	cacheMtx.Lock()
	defer cacheMtx.Unlock()

	cachePath := *repoPath + "/" + path
	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		os.MkdirAll(cachePath, 0755)
		cmd := exec.Command("git", "init", "--bare")
		cmd.Dir = cachePath
		err = cmd.Run()
		if err != nil {
			return err
		}
	}
	return ioutil.WriteFile(cachePath+"/hooks/pre-receive", prereceiveHook, 0755)
}
<commit_msg>gitreceived: Merge pull request #15 from r7kamura/erase<commit_after>package main

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"code.google.com/p/go.crypto/ssh"
	"github.com/flynn/go-shlex"
)

const PrereceiveHookTmpl = `#!/bin/bash
set -eo pipefail; while read oldrev newrev refname; do
[[ $refname = "refs/heads/master" ]] && git archive $newrev | {{RECEIVER}} "$RECEIVE_REPO" "$newrev" | sed -$([[ $(uname) == "Darwin" ]] && echo l || echo u) "s/^/"$'\e[1G\e[K'"/"
done
`

var prereceiveHook []byte

var port = flag.String("p", "22", "port to listen on")
var repoPath = flag.String("r", "/tmp/repos", "path to repo cache")
var noAuth = flag.Bool("n", false, "disable client authentication")
var keys = flag.String("k", "", "pem file containing private keys (read from SSH_PRIVATE_KEYS by default)")

var authChecker []string

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [options] <authchecker> <receiver>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	if flag.NArg() < 2 {
		flag.Usage()
		os.Exit(64)
	}
	var err error
	authChecker, err = shlex.Split(flag.Arg(0))
	if err != nil {
		log.Fatalln("Unable to parse authchecker command:", err)
	}

	receiver, err := shlex.Split(flag.Arg(1))
	if err != nil {
		log.Fatalln("Unable to parse receiver command:", err)
	}
	receiver[0], err = filepath.Abs(receiver[0])
	if err != nil {
		log.Fatalln("Invalid receiver path:", err)
	}
	prereceiveHook = []byte(strings.Replace(PrereceiveHookTmpl, "{{RECEIVER}}", strings.Join(receiver, " "), 1))

	var config *ssh.ServerConfig
	if *noAuth {
		config = &ssh.ServerConfig{NoClientAuth: true}
	} else {
		config = &ssh.ServerConfig{PublicKeyCallback: checkAuth}
	}

	if keyEnv := os.Getenv("SSH_PRIVATE_KEYS"); keyEnv != "" {
		if err := parseKeys(config, []byte(keyEnv)); err != nil {
			log.Fatalln("Failed to load private keys:", err)
		}
	} else {
		pemBytes, err := ioutil.ReadFile(*keys)
		if err != nil {
			log.Fatalln("Failed to load private keys:", err)
		}
		if err := parseKeys(config, pemBytes); err != nil {
			log.Fatalln("Failed to parse private keys:", err)
		}
	}

	if p := os.Getenv("PORT"); p != "" && *port == "22" {
		*port = p
	}
	listener, err := net.Listen("tcp", ":"+*port)
	if err != nil {
		log.Fatalln("Failed to listen for connections:", err)
	}
	for {
		// SSH connections just house multiplexed connections
		conn, err := listener.Accept()
		if err != nil {
			log.Println("Failed to accept incoming connection:", err)
			continue
		}
		go handleConn(conn, config)
	}
}

func parseKeys(conf *ssh.ServerConfig, pemData []byte) error {
	var found bool
	for {
		var block *pem.Block
		block, pemData = pem.Decode(pemData)
		if block == nil {
			if !found {
				return errors.New("no private keys found")
			}
			return nil
		}
		if err := addKey(conf, block); err != nil {
			return err
		}
		found = true
	}
}

func addKey(conf *ssh.ServerConfig, block *pem.Block) (err error) {
	var key interface{}
	switch block.Type {
	case "RSA PRIVATE KEY":
		key, err = x509.ParsePKCS1PrivateKey(block.Bytes)
	case "EC PRIVATE KEY":
		key, err = x509.ParseECPrivateKey(block.Bytes)
	case "DSA PRIVATE KEY":
		key, err = ssh.ParseDSAPrivateKey(block.Bytes)
	default:
		return fmt.Errorf("unsupported key type %q", block.Type)
	}
	if err != nil {
		return err
	}
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		return err
	}
	conf.AddHostKey(signer)
	return nil
}

func handleConn(conn net.Conn, conf *ssh.ServerConfig) {
	defer conn.Close()
	sshConn, chans, reqs, err := ssh.NewServerConn(conn, conf)
	if err != nil {
		log.Println("Failed to handshake:", err)
		return
	}

	go ssh.DiscardRequests(reqs)

	for ch := range chans {
		if ch.ChannelType() != "session" {
			ch.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		go handleChannel(sshConn, ch)
	}
}

func handleChannel(conn *ssh.ServerConn, newChan ssh.NewChannel) {
	ch, reqs, err := newChan.Accept()
	if err != nil {
		log.Println("newChan.Accept failed:", err)
		return
	}
	defer ch.Close()
	for req := range reqs {
		switch req.Type {
		case "exec":
			fail := func(at string, err error) {
				log.Printf("%s failed: %s", at, err)
				ch.Stderr().Write([]byte("Internal error.\n"))
			}
			if req.WantReply {
				req.Reply(true, nil)
			}
			cmdline := string(req.Payload[4:])
			cmdargs, err := shlex.Split(cmdline)
			if err != nil || len(cmdargs) != 2 {
				ch.Stderr().Write([]byte("Invalid arguments.\n"))
				return
			}
			if cmdargs[0] != "git-receive-pack" {
				ch.Stderr().Write([]byte("Only `git push` is supported.\n"))
				return
			}
			cmdargs[1] = strings.TrimSuffix(strings.TrimPrefix(cmdargs[1], "/"), ".git")
			if strings.Contains(cmdargs[1], "..") {
				ch.Stderr().Write([]byte("Invalid repo.\n"))
				return
			}

			if err := ensureCacheRepo(cmdargs[1]); err != nil {
				fail("ensureCacheRepo", err)
				return
			}
			cmd := exec.Command("git-shell", "-c", cmdargs[0]+" '"+cmdargs[1]+"'")
			cmd.Dir = *repoPath
			cmd.Env = append(os.Environ(),
				"RECEIVE_USER="+conn.User(),
				"RECEIVE_REPO="+cmdargs[1],
			)
			done, err := attachCmd(cmd, ch, ch.Stderr(), ch)
			if err != nil {
				fail("attachCmd", err)
				return
			}
			if err := cmd.Start(); err != nil {
				fail("cmd.Start", err)
				return
			}
			done.Wait()
			status, err := exitStatus(cmd.Wait())
			if err != nil {
				fail("exitStatus", err)
				return
			}
			if _, err := ch.SendRequest("exit-status", false, ssh.Marshal(&status)); err != nil {
				fail("sendExit", err)
			}
			return
		case "env":
			if req.WantReply {
				req.Reply(true, nil)
			}
		}
	}
}

var ErrUnauthorized = errors.New("gitreceive: user is unauthorized")

func checkAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
	status, err := exitStatus(exec.Command(authChecker[0],
		append(authChecker[1:], conn.User(), string(bytes.TrimSpace(ssh.MarshalAuthorizedKey(key))))...).Run())
	if err != nil {
		return nil, err
	}
	if status.Status == 0 {
		return nil, nil
	}
	return nil, ErrUnauthorized
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (*sync.WaitGroup, error) {
	var wg sync.WaitGroup
	wg.Add(2)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	go func() {
		io.Copy(stdinIn, stdin)
		stdinIn.Close()
	}()
	go func() {
		io.Copy(stdout, stdoutOut)
		wg.Done()
	}()
	go func() {
		io.Copy(stderr, stderrOut)
		wg.Done()
	}()

	return &wg, nil
}

type exitStatusMsg struct {
	Status uint32
}

func exitStatus(err error) (exitStatusMsg, error) {
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// There is no platform independent way to retrieve
			// the exit code, but the following will work on Unix
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return exitStatusMsg{uint32(status.ExitStatus())}, nil
			}
		}
		return exitStatusMsg{0}, err
	}
	return exitStatusMsg{0}, nil
}

var cacheMtx sync.Mutex

func ensureCacheRepo(path string) error {
	cacheMtx.Lock()
	defer cacheMtx.Unlock()

	cachePath := *repoPath + "/" + path
	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		os.MkdirAll(cachePath, 0755)
		cmd := exec.Command("git", "init", "--bare")
		cmd.Dir = cachePath
		err = cmd.Run()
		if err != nil {
			return err
		}
	}
	return ioutil.WriteFile(cachePath+"/hooks/pre-receive", prereceiveHook, 0755)
}
<|endoftext|>