<commit_before>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// copier.go copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	LookupChannel       chan *DPNResult
	CopyChannel         chan *DPNResult
	PostProcessChannel  chan *DPNResult
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
	RemoteClients       map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

type CopyResult struct {
	LocalPath       string
	ErrorMessage    string
	RsyncStdout     string
	RsyncStderr     string
	InfoMessage     string
	BagWasCopied    bool
}

func NewCopier(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Copier, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)

	// HACK: ProcUtil assumes the volume it should be managing is
	// the APTrust volume. In this case, it's the DPN volume.
	// DPNStagingDirectory
	volume, err := bagman.NewVolume(procUtil.Config.DPNStagingDirectory, procUtil.MessageLog)
	if err != nil {
		message := fmt.Sprintf("Exiting. Cannot init Volume object: %v", err)
		fmt.Fprintln(os.Stderr, message)
		procUtil.MessageLog.Fatal(message)
	}
	procUtil.Volume = volume

	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := GetRemoteClients(localClient, dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	copier.LookupChannel = make(chan *DPNResult, workerBufferSize)
	copier.CopyChannel = make(chan *DPNResult, workerBufferSize)
	copier.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go copier.doLookup()
		go copier.doCopy()
		go copier.postProcess()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	dpnResult := &DPNResult{}
	err := json.Unmarshal(message.Body, dpnResult)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq. " +
			"Error is: %v    JSON is: %s", err.Error(), string(message.Body))
		copier.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Set up the copy result
	dpnResult.CopyResult = &CopyResult{
		BagWasCopied: false,
	}
	dpnResult.NsqMessage = message

	// Start processing.
	dpnResult.Stage = STAGE_COPY
	copier.LookupChannel <- dpnResult
	copier.ProcUtil.MessageLog.Info("Put %s from %s into copy channel",
		dpnResult.DPNBag.UUID, dpnResult.DPNBag.AdminNode)
	return nil
}

// Look up the DPN bag on the admin node. Although we already have the
// bag object as bart of the DPNResult object, this request may have been
// sitting in the queue for many hours, and the replication request may
// have been fulfilled or cancelled in that time. So check the status on
// the authoritative node to avoid unnecessarily processing what might
// be hundreds of gigs of data.
func (copier *Copier) doLookup() {
	for result := range copier.LookupChannel {
		// Get a client to talk to the FromNode
		remoteClient := copier.RemoteClients[result.TransferRequest.FromNode]

		copier.ProcUtil.MessageLog.Debug(
			"Looking up ReplicationId %s, bag %s, on node %s ",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode)

		// If we can find out for sure that this replication request should
		// not be processed, then don't process it...
		xfer, _ := remoteClient.ReplicationTransferGet(
			result.TransferRequest.ReplicationId)
		if xfer != nil && xfer.Status != "requested" {
			message := fmt.Sprintf(
				"Cancelling copy of ReplicationId %s (bag %s) because " +
					"replication status on %s is %s",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode,
				xfer.Status)
			copier.ProcUtil.MessageLog.Info(message)
			result.CopyResult.InfoMessage = message
			result.Retry = false
			result.TransferRequest = xfer
			copier.PostProcessChannel <- result
			continue
		}
		// ...otherwise, proceed with processing.
		copier.CopyChannel <- result
	}
}

// Copy the file from the remote node to our local staging area.
// Calculate checksums.
func (copier *Copier) doCopy() {
	for result := range copier.CopyChannel {

		// Make sure we have enough room on the volume
		// to download and unpack this bag.
		err := copier.ProcUtil.Volume.Reserve(uint64(float64(result.DPNBag.Size) * float64(2.1)))
		if err != nil {
			// Not enough room on disk
			msg := fmt.Sprintf(
				"Requeueing %s from %s (%d bytes) - not enough disk space",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.FromNode,
				result.DPNBag.Size)
			copier.ProcUtil.MessageLog.Warning(msg)
			if result.NsqMessage != nil {
				result.NsqMessage.Requeue(1 * time.Hour)
			} else {
				result.ErrorMessage = msg
				result.CopyResult.ErrorMessage = msg
				copier.WaitGroup.Done()
			}
			continue
		}

		localPath := filepath.Join(
			copier.ProcUtil.Config.DPNStagingDirectory,
			fmt.Sprintf("%s.tar", result.TransferRequest.BagId))

		if !bagman.FileExists(copier.ProcUtil.Config.DPNStagingDirectory) {
			os.MkdirAll(copier.ProcUtil.Config.DPNStagingDirectory, 0755)
		}

		// DEBUG - use for tracing 'file not found'
		// fmt.Printf("Rsync link is %s\n", result.TransferRequest.Link)

		copier.ProcUtil.MessageLog.Info("Rsync link is %s", result.TransferRequest.Link)
		rsyncCommand := GetRsyncCommand(result.TransferRequest.Link,
			localPath, copier.DPNConfig.UseSSHWithRsync)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		if err != nil {
			result.CopyResult.ErrorMessage = fmt.Sprintf("%s: %s",
				err.Error(), string(output))
		} else {
			result.LocalPath = localPath
			result.CopyResult.LocalPath = localPath
			result.CopyResult.BagWasCopied = true
			// TODO: This is not necessary. We just need to calculate the checksum
			// of the SHA-256 manifest
			fileDigest, err := bagman.CalculateDigests(localPath)
			if result.NsqMessage != nil {
				result.NsqMessage.Touch()
			}
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Could not calculate checksums on '%s': %v",
					result.PackageResult.TarFilePath, err)
				copier.PostProcessChannel <- result
				continue
			}
			result.BagMd5Digest = fileDigest.Md5Digest
			result.BagSha256Digest = fileDigest.Sha256Digest
			result.BagSize = fileDigest.Size
		}
		copier.PostProcessChannel <- result
	}
}

func (copier *Copier) postProcess() {
	// On success, send to validation queue.
	// Otherwise, send to trouble queue.
	for result := range copier.PostProcessChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		result.ErrorMessage = result.CopyResult.ErrorMessage

		// On error, log and send to trouble queue if the error is fatal
		if result.ErrorMessage != "" {
			copier.ProcUtil.MessageLog.Error(result.ErrorMessage)
			copier.ProcUtil.IncrementFailed()
			if result.Retry == false {
				SendToTroubleQueue(result, copier.ProcUtil)
			}
			if bagman.FileExists(result.CopyResult.LocalPath) {
				os.Remove(result.CopyResult.LocalPath)
				copier.ProcUtil.MessageLog.Debug(
					"Deleting bag file %s", result.CopyResult.LocalPath)
			}
		} else if result.CopyResult.BagWasCopied == false {
			// We didn't copy the bag, but there was no error.
			// This happens when the transfer request is marked
			// as completed or cancelled on the remote node.
			// Count this as success, because we did what we're
			// supposed to do in this case, which is nothing.
			copier.ProcUtil.IncrementSucceeded()
		} else {
			// We successfully copied the bag. Send it on to
			// the validation queue.
			copier.ProcUtil.IncrementSucceeded()
			SendToValidationQueue(result, copier.ProcUtil)
		}

		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			copier.WaitGroup.Done()
		} else {
			result.NsqMessage.Finish()
		}
		copier.ProcUtil.LogStats()

	}
}

func (copier *Copier) RunTest(dpnResult *DPNResult) {
	copier.WaitGroup.Add(1)
	copier.ProcUtil.MessageLog.Info("Putting %s into lookup channel",
		dpnResult.BagIdentifier)
	copier.CopyChannel <- dpnResult
	copier.WaitGroup.Wait()
}

// Returns a command object for copying from the remote location to
// the local filesystem. The copy is done via rsync over ssh, and
// the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }

//
func GetRsyncCommand(copyFrom, copyTo string, useSSH bool) (*exec.Cmd) {
//	rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	if useSSH {
		return exec.Command("rsync", "-avz", "-e",  "ssh", copyFrom, copyTo)
	}
	return exec.Command("rsync", "-avz", copyFrom, copyTo)
}
<commit_msg>Log specific error from volume manager<commit_after>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// copier.go copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	LookupChannel       chan *DPNResult
	CopyChannel         chan *DPNResult
	PostProcessChannel  chan *DPNResult
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
	RemoteClients       map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

type CopyResult struct {
	LocalPath       string
	ErrorMessage    string
	RsyncStdout     string
	RsyncStderr     string
	InfoMessage     string
	BagWasCopied    bool
}

func NewCopier(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Copier, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)

	// HACK: ProcUtil assumes the volume it should be managing is
	// the APTrust volume. In this case, it's the DPN volume.
	// DPNStagingDirectory
	volume, err := bagman.NewVolume(procUtil.Config.DPNStagingDirectory, procUtil.MessageLog)
	if err != nil {
		message := fmt.Sprintf("Exiting. Cannot init Volume object: %v", err)
		fmt.Fprintln(os.Stderr, message)
		procUtil.MessageLog.Fatal(message)
	}
	procUtil.Volume = volume

	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := GetRemoteClients(localClient, dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	copier.LookupChannel = make(chan *DPNResult, workerBufferSize)
	copier.CopyChannel = make(chan *DPNResult, workerBufferSize)
	copier.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go copier.doLookup()
		go copier.doCopy()
		go copier.postProcess()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	dpnResult := &DPNResult{}
	err := json.Unmarshal(message.Body, dpnResult)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq. " +
			"Error is: %v    JSON is: %s", err.Error(), string(message.Body))
		copier.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Set up the copy result
	dpnResult.CopyResult = &CopyResult{
		BagWasCopied: false,
	}
	dpnResult.NsqMessage = message

	// Start processing.
	dpnResult.Stage = STAGE_COPY
	copier.LookupChannel <- dpnResult
	copier.ProcUtil.MessageLog.Info("Put %s from %s into copy channel",
		dpnResult.DPNBag.UUID, dpnResult.DPNBag.AdminNode)
	return nil
}

// Look up the DPN bag on the admin node. Although we already have the
// bag object as bart of the DPNResult object, this request may have been
// sitting in the queue for many hours, and the replication request may
// have been fulfilled or cancelled in that time. So check the status on
// the authoritative node to avoid unnecessarily processing what might
// be hundreds of gigs of data.
func (copier *Copier) doLookup() {
	for result := range copier.LookupChannel {
		// Get a client to talk to the FromNode
		remoteClient := copier.RemoteClients[result.TransferRequest.FromNode]

		copier.ProcUtil.MessageLog.Debug(
			"Looking up ReplicationId %s, bag %s, on node %s ",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode)

		// If we can find out for sure that this replication request should
		// not be processed, then don't process it...
		xfer, _ := remoteClient.ReplicationTransferGet(
			result.TransferRequest.ReplicationId)
		if xfer != nil && xfer.Status != "requested" {
			message := fmt.Sprintf(
				"Cancelling copy of ReplicationId %s (bag %s) because " +
					"replication status on %s is %s",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.BagId,
				result.TransferRequest.FromNode,
				xfer.Status)
			copier.ProcUtil.MessageLog.Info(message)
			result.CopyResult.InfoMessage = message
			result.Retry = false
			result.TransferRequest = xfer
			copier.PostProcessChannel <- result
			continue
		}
		// ...otherwise, proceed with processing.
		copier.CopyChannel <- result
	}
}

// Copy the file from the remote node to our local staging area.
// Calculate checksums.
func (copier *Copier) doCopy() {
	for result := range copier.CopyChannel {

		// Make sure we have enough room on the volume
		// to download and unpack this bag.
		err := copier.ProcUtil.Volume.Reserve(uint64(float64(result.DPNBag.Size) * float64(2.1)))
		if err != nil {
			// Not enough room on disk
			msg := fmt.Sprintf(
				"Requeueing %s from %s (%d bytes) - not enough disk space: %v",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.FromNode,
				result.DPNBag.Size, err)
			copier.ProcUtil.MessageLog.Warning(msg)
			if result.NsqMessage != nil {
				result.NsqMessage.Requeue(1 * time.Hour)
			} else {
				result.ErrorMessage = msg
				result.CopyResult.ErrorMessage = msg
				copier.WaitGroup.Done()
			}
			continue
		}

		localPath := filepath.Join(
			copier.ProcUtil.Config.DPNStagingDirectory,
			fmt.Sprintf("%s.tar", result.TransferRequest.BagId))

		if !bagman.FileExists(copier.ProcUtil.Config.DPNStagingDirectory) {
			os.MkdirAll(copier.ProcUtil.Config.DPNStagingDirectory, 0755)
		}

		// DEBUG - use for tracing 'file not found'
		// fmt.Printf("Rsync link is %s\n", result.TransferRequest.Link)

		copier.ProcUtil.MessageLog.Info("Rsync link is %s", result.TransferRequest.Link)
		rsyncCommand := GetRsyncCommand(result.TransferRequest.Link,
			localPath, copier.DPNConfig.UseSSHWithRsync)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		if err != nil {
			result.CopyResult.ErrorMessage = fmt.Sprintf("%s: %s",
				err.Error(), string(output))
		} else {
			result.LocalPath = localPath
			result.CopyResult.LocalPath = localPath
			result.CopyResult.BagWasCopied = true
			// TODO: This is not necessary. We just need to calculate the checksum
			// of the SHA-256 manifest
			fileDigest, err := bagman.CalculateDigests(localPath)
			if result.NsqMessage != nil {
				result.NsqMessage.Touch()
			}
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Could not calculate checksums on '%s': %v",
					result.PackageResult.TarFilePath, err)
				copier.PostProcessChannel <- result
				continue
			}
			result.BagMd5Digest = fileDigest.Md5Digest
			result.BagSha256Digest = fileDigest.Sha256Digest
			result.BagSize = fileDigest.Size
		}
		copier.PostProcessChannel <- result
	}
}

func (copier *Copier) postProcess() {
	// On success, send to validation queue.
	// Otherwise, send to trouble queue.
	for result := range copier.PostProcessChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		result.ErrorMessage = result.CopyResult.ErrorMessage

		// On error, log and send to trouble queue if the error is fatal
		if result.ErrorMessage != "" {
			copier.ProcUtil.MessageLog.Error(result.ErrorMessage)
			copier.ProcUtil.IncrementFailed()
			if result.Retry == false {
				SendToTroubleQueue(result, copier.ProcUtil)
			}
			if bagman.FileExists(result.CopyResult.LocalPath) {
				os.Remove(result.CopyResult.LocalPath)
				copier.ProcUtil.MessageLog.Debug(
					"Deleting bag file %s", result.CopyResult.LocalPath)
			}
		} else if result.CopyResult.BagWasCopied == false {
			// We didn't copy the bag, but there was no error.
			// This happens when the transfer request is marked
			// as completed or cancelled on the remote node.
			// Count this as success, because we did what we're
			// supposed to do in this case, which is nothing.
			copier.ProcUtil.IncrementSucceeded()
		} else {
			// We successfully copied the bag. Send it on to
			// the validation queue.
			copier.ProcUtil.IncrementSucceeded()
			SendToValidationQueue(result, copier.ProcUtil)
		}

		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			copier.WaitGroup.Done()
		} else {
			result.NsqMessage.Finish()
		}
		copier.ProcUtil.LogStats()

	}
}

func (copier *Copier) RunTest(dpnResult *DPNResult) {
	copier.WaitGroup.Add(1)
	copier.ProcUtil.MessageLog.Info("Putting %s into lookup channel",
		dpnResult.BagIdentifier)
	copier.CopyChannel <- dpnResult
	copier.WaitGroup.Wait()
}

// Returns a command object for copying from the remote location to
// the local filesystem. The copy is done via rsync over ssh, and
// the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }

//
func GetRsyncCommand(copyFrom, copyTo string, useSSH bool) (*exec.Cmd) {
//	rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	if useSSH {
		return exec.Command("rsync", "-avz", "-e",  "ssh", copyFrom, copyTo)
	}
	return exec.Command("rsync", "-avz", copyFrom, copyTo)
}
<|endoftext|>


<commit_before>package strutils

import (
	"reflect"
	"runtime"
	"strings"
	"sync"
	"testing"
)

// Assert is Methods for helping testing the strutils pkg.
type Assert struct {
	mutx         sync.RWMutex
	plib         *StringProc
	unitTestMode bool
}

// NewAssert Creates and returns a String processing methods's pointer.
func NewAssert() *Assert {
	obj := &Assert{}
	obj.plib = NewStringProc()

	obj.unitTestMode = UNITTESTMODE

	return obj
}

// TurnOnUnitTestMode is turn on unitTestMode
func (a *Assert) TurnOnUnitTestMode() {
	a.mutx.Lock()
	defer a.mutx.Unlock()
	a.unitTestMode = true
}

// TurnOffUnitTestMode is turn off unitTestMode
func (a *Assert) TurnOffUnitTestMode() {
	a.mutx.Lock()
	defer a.mutx.Unlock()
	a.unitTestMode = false
}

// printMsg is equivalent to t.Errorf include other information for easy debug
func (a *Assert) printMsg(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	outf := t.Errorf
	out := t.Error

	if !a.unitTestMode { // assertion methods test on go test
		outf = t.Logf
		out = t.Log
		t.Log("*** The following message is just failure testing ***")
	}

	funcn, file, line, _ := runtime.Caller(2)
	out(strings.Repeat("-", 120))
	outf("+ %v:%v\n", file, line)
	outf("+ %+v\n", runtime.FuncForPC(funcn).Name())
	out(strings.Repeat("-", 120))
	outf(msgfmt, args...)

	outf("- value1 : %+v\n", v1)
	if v2 != nil {
		outf("- value2 : %+v\n", v2)
	}

	out(strings.Repeat("-", 120))
}

/*
//TODO: will remove below, numericTypeUpCase better than below
//isCompareableNum asserts the specified objects are can compareble
func (a *Assert) isComparableNum(t *testing.T, v1 interface{}, v2 interface{}) bool {

	if !reflect.ValueOf(v1).IsValid() || !reflect.ValueOf(v2).IsValid() {
		a.printMsg(t, v1, v2, "Invalid Value")
		return false
	}

	refv1 := reflect.TypeOf(v1)
	refv2 := reflect.TypeOf(v2)

	if refv1.Comparable() != refv2.Comparable() {
		a.printMsg(t, v1, v2, "Not Comparable")
		return false
	}

	refv1k := refv1.Kind()
	refv2k := refv2.Kind()

	//int ~ int64 (0x2 ~ 0x6)
	//uint ~ uint64 (0x7 ~ 0xc)
	//float ~ float64 (0xd ~ 0xe)
	if (refv1k >= 0x2 && refv1k <= 0xe) && (refv2k >= 0x2 && refv2k <= 0xe) {
		return true
	}

	a.printMsg(t, v1, v2, "Different Type v1.(%+v) != v2(%+v)", refv1k, refv2k)
	return false
}
*/

// numericTypeUpCase converts the any numeric type to upsize type of that
func (a *Assert) numericTypeUpCase(val interface{}) (int64, uint64, float64, bool) {
	var tmpint int64
	var tmpuint uint64
	var tmpfloat float64

	switch val.(type) {
	case int:
		tmpint = int64(val.(int))
	case int8:
		tmpint = int64(val.(int8))
	case int16:
		tmpint = int64(val.(int16))
	case int32:
		tmpint = int64(val.(int32))
	case int64:
		tmpint = val.(int64)
	case uint:
		tmpuint = uint64(val.(uint))
	case uint8:
		tmpuint = uint64(val.(uint8))
	case uint16:
		tmpuint = uint64(val.(uint16))
	case uint32:
		tmpuint = uint64(val.(uint32))
	case uint64:
		tmpuint = val.(uint64)
	case float32:
		tmpfloat = float64(val.(float32))
	case float64:
		tmpfloat = val.(float64)
	default:
		return 0, 0, 0, false
	}

	return tmpint, tmpuint, tmpfloat, true
}

// AssertLog formats its arguments using default formatting, analogous to t.Log
func (a *Assert) AssertLog(t *testing.T, err error, msgfmt string, args ...interface{}) {
	if err != nil {
		t.Logf("Error : %v", err)
	}

	if len(args) > 0 {
		t.Logf(msgfmt, args...)
	} else {
		t.Log(msgfmt)
	}
}

// AssertEquals asserts that two objects are equal.
func (a *Assert) AssertEquals(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	ok, err := a.plib.AnyCompare(v1, v2)
	if err != nil {
		a.printMsg(t, v1, v2, err.Error())
	}

	if !ok {
		a.printMsg(t, v1, v2, "not compare")
	}
}

// AssertNotEquals asserts that two objects are not equal.
func (a *Assert) AssertNotEquals(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	_, err := a.plib.AnyCompare(v1, v2)
	if err == nil && v1 == v2 {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertFalse asserts that the specified value is false.
func (a *Assert) AssertFalse(t *testing.T, v1 bool, msgfmt string, args ...interface{}) {
	if v1 {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertTrue asserts that the specified value is true.
func (a *Assert) AssertTrue(t *testing.T, v1 bool, msgfmt string, args ...interface{}) {
	if !v1 {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertNil asserts that the specified value is nil.
func (a *Assert) AssertNil(t *testing.T, v1 interface{}, msgfmt string, args ...interface{}) {
	if v1 != nil {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertNotNil asserts that the specified value isn't nil.
func (a *Assert) AssertNotNil(t *testing.T, v1 interface{}, msgfmt string, args ...interface{}) {
	if v1 == nil {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertLessThan asserts that the specified value are v1 less than v2
func (a *Assert) AssertLessThan(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	var retval bool

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int < tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint < tmpv2uint)
	case float32, float64:
		retval = (tmpv1float < tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertLessThanEqualTo asserts that the specified value are v1 less than v2 or equal to
func (a *Assert) AssertLessThanEqualTo(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	var retval bool

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int <= tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint <= tmpv2uint)
	case float32, float64:
		retval = (tmpv1float <= tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertGreaterThan nsserts that the specified value are v1 greater than v2
func (a *Assert) AssertGreaterThan(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	retval := false

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int > tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint > tmpv2uint)
	case float32, float64:
		retval = (tmpv1float > tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertGreaterThanEqualTo asserts that the specified value are v1 greater than v2 or equal to
func (a *Assert) AssertGreaterThanEqualTo(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	retval := false

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int >= tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint >= tmpv2uint)
	case float32, float64:
		retval = (tmpv1float >= tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertLengthOf asserts that object has a length property with the expected value.
func (a *Assert) AssertLengthOf(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	var tmplen int
	retval := false

	v1val := reflect.ValueOf(v1)

	switch v1val.Kind() {

	case reflect.String:
		tmplen = len(v1val.String())
	case reflect.Array, reflect.Chan, reflect.Slice:
		tmplen = v1val.Len()
	case reflect.Map:
		tmplen = len(v1val.MapKeys())
	default:
		a.printMsg(t, v1, v2, "Required data type of value 1 must be countable data-type (String,Array,Chan,Map,Slice)")
		return
	}

	if tmplen < 0 {
		a.printMsg(t, v1, v2, "Failured, can't count the value 1([%s]=%v)", v1val.Kind().String(), v1)
		return
	}

	tmpv2int, tmpv2uint, _, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv1int := int64(tmplen)
	tmpv1uint := uint64(tmplen)

	switch v2.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int == tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint == tmpv2uint)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}
<commit_msg>Fix : Line 70: warning: "compareble" is a misspelling of "comparable" (misspell)<commit_after>package strutils

import (
	"reflect"
	"runtime"
	"strings"
	"sync"
	"testing"
)

// Assert is Methods for helping testing the strutils pkg.
type Assert struct {
	mutx         sync.RWMutex
	plib         *StringProc
	unitTestMode bool
}

// NewAssert Creates and returns a String processing methods's pointer.
func NewAssert() *Assert {
	obj := &Assert{}
	obj.plib = NewStringProc()

	obj.unitTestMode = UNITTESTMODE

	return obj
}

// TurnOnUnitTestMode is turn on unitTestMode
func (a *Assert) TurnOnUnitTestMode() {
	a.mutx.Lock()
	defer a.mutx.Unlock()
	a.unitTestMode = true
}

// TurnOffUnitTestMode is turn off unitTestMode
func (a *Assert) TurnOffUnitTestMode() {
	a.mutx.Lock()
	defer a.mutx.Unlock()
	a.unitTestMode = false
}

// printMsg is equivalent to t.Errorf include other information for easy debug
func (a *Assert) printMsg(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	outf := t.Errorf
	out := t.Error

	if !a.unitTestMode { // assertion methods test on go test
		outf = t.Logf
		out = t.Log
		t.Log("*** The following message is just failure testing ***")
	}

	funcn, file, line, _ := runtime.Caller(2)
	out(strings.Repeat("-", 120))
	outf("+ %v:%v\n", file, line)
	outf("+ %+v\n", runtime.FuncForPC(funcn).Name())
	out(strings.Repeat("-", 120))
	outf(msgfmt, args...)

	outf("- value1 : %+v\n", v1)
	if v2 != nil {
		outf("- value2 : %+v\n", v2)
	}

	out(strings.Repeat("-", 120))
}

/*
//TODO: will remove below, numericTypeUpCase better than below
//isComparableNum asserts the specified objects are can compareble
func (a *Assert) isComparableNum(t *testing.T, v1 interface{}, v2 interface{}) bool {

	if !reflect.ValueOf(v1).IsValid() || !reflect.ValueOf(v2).IsValid() {
		a.printMsg(t, v1, v2, "Invalid Value")
		return false
	}

	refv1 := reflect.TypeOf(v1)
	refv2 := reflect.TypeOf(v2)

	if refv1.Comparable() != refv2.Comparable() {
		a.printMsg(t, v1, v2, "Not Comparable")
		return false
	}

	refv1k := refv1.Kind()
	refv2k := refv2.Kind()

	//int ~ int64 (0x2 ~ 0x6)
	//uint ~ uint64 (0x7 ~ 0xc)
	//float ~ float64 (0xd ~ 0xe)
	if (refv1k >= 0x2 && refv1k <= 0xe) && (refv2k >= 0x2 && refv2k <= 0xe) {
		return true
	}

	a.printMsg(t, v1, v2, "Different Type v1.(%+v) != v2(%+v)", refv1k, refv2k)
	return false
}
*/

// numericTypeUpCase converts the any numeric type to upsize type of that
func (a *Assert) numericTypeUpCase(val interface{}) (int64, uint64, float64, bool) {
	var tmpint int64
	var tmpuint uint64
	var tmpfloat float64

	switch val.(type) {
	case int:
		tmpint = int64(val.(int))
	case int8:
		tmpint = int64(val.(int8))
	case int16:
		tmpint = int64(val.(int16))
	case int32:
		tmpint = int64(val.(int32))
	case int64:
		tmpint = val.(int64)
	case uint:
		tmpuint = uint64(val.(uint))
	case uint8:
		tmpuint = uint64(val.(uint8))
	case uint16:
		tmpuint = uint64(val.(uint16))
	case uint32:
		tmpuint = uint64(val.(uint32))
	case uint64:
		tmpuint = val.(uint64)
	case float32:
		tmpfloat = float64(val.(float32))
	case float64:
		tmpfloat = val.(float64)
	default:
		return 0, 0, 0, false
	}

	return tmpint, tmpuint, tmpfloat, true
}

// AssertLog formats its arguments using default formatting, analogous to t.Log
func (a *Assert) AssertLog(t *testing.T, err error, msgfmt string, args ...interface{}) {
	if err != nil {
		t.Logf("Error : %v", err)
	}

	if len(args) > 0 {
		t.Logf(msgfmt, args...)
	} else {
		t.Log(msgfmt)
	}
}

// AssertEquals asserts that two objects are equal.
func (a *Assert) AssertEquals(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	ok, err := a.plib.AnyCompare(v1, v2)
	if err != nil {
		a.printMsg(t, v1, v2, err.Error())
	}

	if !ok {
		a.printMsg(t, v1, v2, "not compare")
	}
}

// AssertNotEquals asserts that two objects are not equal.
func (a *Assert) AssertNotEquals(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	_, err := a.plib.AnyCompare(v1, v2)
	if err == nil && v1 == v2 {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertFalse asserts that the specified value is false.
func (a *Assert) AssertFalse(t *testing.T, v1 bool, msgfmt string, args ...interface{}) {
	if v1 {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertTrue asserts that the specified value is true.
func (a *Assert) AssertTrue(t *testing.T, v1 bool, msgfmt string, args ...interface{}) {
	if !v1 {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertNil asserts that the specified value is nil.
func (a *Assert) AssertNil(t *testing.T, v1 interface{}, msgfmt string, args ...interface{}) {
	if v1 != nil {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertNotNil asserts that the specified value isn't nil.
func (a *Assert) AssertNotNil(t *testing.T, v1 interface{}, msgfmt string, args ...interface{}) {
	if v1 == nil {
		a.printMsg(t, v1, nil, msgfmt, args...)
	}
}

// AssertLessThan asserts that the specified value are v1 less than v2
func (a *Assert) AssertLessThan(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	var retval bool

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int < tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint < tmpv2uint)
	case float32, float64:
		retval = (tmpv1float < tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertLessThanEqualTo asserts that the specified value are v1 less than v2 or equal to
func (a *Assert) AssertLessThanEqualTo(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	var retval bool

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int <= tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint <= tmpv2uint)
	case float32, float64:
		retval = (tmpv1float <= tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertGreaterThan nsserts that the specified value are v1 greater than v2
func (a *Assert) AssertGreaterThan(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	retval := false

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int > tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint > tmpv2uint)
	case float32, float64:
		retval = (tmpv1float > tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertGreaterThanEqualTo asserts that the specified value are v1 greater than v2 or equal to
func (a *Assert) AssertGreaterThanEqualTo(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	/*
		if !a.isComparableNum(t, v1, v2) {
			return
		}
	*/

	tmpv1int, tmpv1uint, tmpv1float, ok := a.numericTypeUpCase(v1)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 1 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv2int, tmpv2uint, tmpv2float, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	retval := false

	switch v1.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int >= tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint >= tmpv2uint)
	case float32, float64:
		retval = (tmpv1float >= tmpv2float)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}

// AssertLengthOf asserts that object has a length property with the expected value.
func (a *Assert) AssertLengthOf(t *testing.T, v1 interface{}, v2 interface{}, msgfmt string, args ...interface{}) {
	var tmplen int
	retval := false

	v1val := reflect.ValueOf(v1)

	switch v1val.Kind() {

	case reflect.String:
		tmplen = len(v1val.String())
	case reflect.Array, reflect.Chan, reflect.Slice:
		tmplen = v1val.Len()
	case reflect.Map:
		tmplen = len(v1val.MapKeys())
	default:
		a.printMsg(t, v1, v2, "Required data type of value 1 must be countable data-type (String,Array,Chan,Map,Slice)")
		return
	}

	if tmplen < 0 {
		a.printMsg(t, v1, v2, "Failured, can't count the value 1([%s]=%v)", v1val.Kind().String(), v1)
		return
	}

	tmpv2int, tmpv2uint, _, ok := a.numericTypeUpCase(v2)
	if !ok {
		a.printMsg(t, v1, v2, "Required value 2 must be a numeric (int,uint,float with bit (8~64)")
		return
	}

	tmpv1int := int64(tmplen)
	tmpv1uint := uint64(tmplen)

	switch v2.(type) {
	case int, int8, int16, int32, int64:
		retval = (tmpv1int == tmpv2int)
	case uint, uint8, uint16, uint32, uint64:
		retval = (tmpv1uint == tmpv2uint)
	}

	if !retval {
		a.printMsg(t, v1, v2, msgfmt, args...)
	}
}
<|endoftext|>


<commit_before>// Description: conf log 实现日志相关的配置项
// Author: ZHU HAIHUA
// Since: 2016-02-27 15:10
package conf

import (
	"fmt"
	"github.com/gin-gonic/gin"
	log "github.com/kimiazhu/log4go"
	"time"
)

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func ErrorLoggerT(typ gin.ErrorType) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Next()
		// avoid writting if we already wrote into the response body
		if !c.Writer.Written() {
			errors := c.Errors.ByType(typ)
			if len(errors) > 0 {
				c.JSON(-1, errors)
			}
		}
	}
}

func AccessLog() gin.HandlerFunc {
	return func(c *gin.Context) {
		// Start timer
		start := time.Now()
		path := c.Request.URL.Path

		// Process request
		c.Next()

		end := time.Now()
		latency := end.Sub(start)

		clientIP := c.ClientIP()
		method := c.Request.Method
		statusCode := c.Writer.Status()
		statusColor := colorForStatus(statusCode)
		methodColor := colorForMethod(method)
		comment := c.Errors.ByType(gin.ErrorTypePrivate).String()

		log.Access(fmt.Sprintf("%v | %s  %s %-7s %s | %s %3d %s | %13v | %s | %s",
			start.Format("2006/01/02 15:04:05.999999999"),
			methodColor, reset, method, path,
			statusColor, statusCode, reset,
			latency,
			clientIP,
			comment,
		))
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<commit_msg>change time format of access log<commit_after>// Description: conf log 实现日志相关的配置项
// Author: ZHU HAIHUA
// Since: 2016-02-27 15:10
package conf

import (
	"fmt"
	"github.com/gin-gonic/gin"
	log "github.com/kimiazhu/log4go"
	"time"
)

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func ErrorLoggerT(typ gin.ErrorType) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Next()
		// avoid writting if we already wrote into the response body
		if !c.Writer.Written() {
			errors := c.Errors.ByType(typ)
			if len(errors) > 0 {
				c.JSON(-1, errors)
			}
		}
	}
}

func AccessLog() gin.HandlerFunc {
	return func(c *gin.Context) {
		// Start timer
		start := time.Now()
		path := c.Request.URL.Path

		// Process request
		c.Next()

		end := time.Now()
		latency := end.Sub(start)

		clientIP := c.ClientIP()
		method := c.Request.Method
		statusCode := c.Writer.Status()
		statusColor := colorForStatus(statusCode)
		methodColor := colorForMethod(method)
		comment := c.Errors.ByType(gin.ErrorTypePrivate).String()

		log.Access(fmt.Sprintf("%v | %s  %s %-7s %s | %s %3d %s | %13v | %s | %s",
			start.Format("2006/01/02 15:04:05.000000000"),
			methodColor, reset, method, path,
			statusColor, statusCode, reset,
			latency,
			clientIP,
			comment,
		))
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<|endoftext|>


<commit_before>package relay

import (
	"context"
	"fmt"
	"math/rand"
	"sync"
	"time"

	basic "github.com/libp2p/go-libp2p/p2p/host/basic"

	autonat "github.com/libp2p/go-libp2p-autonat"
	discovery "github.com/libp2p/go-libp2p-discovery"
	host "github.com/libp2p/go-libp2p-host"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

var (
	DesiredRelays = 3

	BootDelay = 90 * time.Second
)

// AutoRelayHost is a Host that uses relays for connectivity when a NAT is detected.
type AutoRelayHost struct {
	*basic.BasicHost
	discover discovery.Discoverer
	autonat  autonat.AutoNAT
	addrsF   basic.AddrsFactory

	disconnect chan struct{}

	mx     sync.Mutex
	relays map[peer.ID]pstore.PeerInfo
	addrs  []ma.Multiaddr
}

func NewAutoRelayHost(ctx context.Context, bhost *basic.BasicHost, discover discovery.Discoverer) *AutoRelayHost {
	autonat := autonat.NewAutoNAT(ctx, bhost, bhost.AllAddrs)
	h := &AutoRelayHost{
		BasicHost:  bhost,
		discover:   discover,
		autonat:    autonat,
		addrsF:     bhost.AddrsFactory,
		relays:     make(map[peer.ID]pstore.PeerInfo),
		disconnect: make(chan struct{}, 1),
	}
	bhost.AddrsFactory = h.hostAddrs
	bhost.Network().Notify(h)
	go h.background(ctx)
	return h
}

func (h *AutoRelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr {
	h.mx.Lock()
	defer h.mx.Unlock()
	if h.addrs != nil && h.autonat.Status() == autonat.NATStatusPrivate {
		return h.addrs
	} else {
		return h.addrsF(addrs)
	}
}

func (h *AutoRelayHost) background(ctx context.Context) {
	select {
	case <-time.After(autonat.AutoNATBootDelay + BootDelay):
	case <-ctx.Done():
		return
	}

	for {
		wait := autonat.AutoNATRefreshInterval
		switch h.autonat.Status() {
		case autonat.NATStatusUnknown:
			wait = autonat.AutoNATRetryInterval
		case autonat.NATStatusPublic:
		case autonat.NATStatusPrivate:
			h.findRelays(ctx)
		}

		select {
		case <-h.disconnect:
			// invalidate addrs
			h.mx.Lock()
			h.addrs = nil
			h.mx.Unlock()
		case <-time.After(wait):
		case <-ctx.Done():
			return
		}
	}
}

func (h *AutoRelayHost) findRelays(ctx context.Context) {
	h.mx.Lock()
	if len(h.relays) >= DesiredRelays {
		h.mx.Unlock()
		return
	}
	need := DesiredRelays - len(h.relays)
	h.mx.Unlock()

	limit := 20
	for ; need > limit; limit *= 2 {
	}

	dctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	pis, err := discovery.FindPeers(dctx, h.discover, "/libp2p/relay", limit)
	cancel()
	if err != nil {
		log.Debugf("error discovering relays: %s", err.Error())
		return
	}

	// TODO better relay selection strategy; this just selects random relays
	//      but we should probably use ping latency as the selection metric
	shuffleRelays(pis)

	update := 0

	for _, pi := range pis {
		h.mx.Lock()
		if _, ok := h.relays[pi.ID]; ok {
			h.mx.Unlock()
			continue
		}
		h.mx.Unlock()

		cctx, cancel := context.WithTimeout(ctx, 60*time.Second)
		err = h.Connect(cctx, pi)
		cancel()
		if err != nil {
			log.Debugf("error connecting to relay %s: %s", pi.ID, err.Error())
			continue
		}

		log.Debugf("connected to relay %s", pi.ID)
		h.mx.Lock()
		h.relays[pi.ID] = pi
		h.mx.Unlock()

		update++
		need--
		if need == 0 {
			break
		}
	}

	if update > 0 || h.addrs == nil {
		h.updateAddrs()
	}
}

func (h *AutoRelayHost) updateAddrs() {
	h.doUpdateAddrs()
	h.PushIdentify()
}

func (h *AutoRelayHost) doUpdateAddrs() {
	h.mx.Lock()
	defer h.mx.Unlock()

	addrs := h.addrsF(h.AllAddrs())
	raddrs := make([]ma.Multiaddr, 0, len(addrs)+len(h.relays))

	// remove our public addresses from the list and replace them by just the public IP
	for _, addr := range addrs {
		if manet.IsPublicAddr(addr) {
			ip, err := addr.ValueForProtocol(ma.P_IP4)
			if err == nil {
				pub, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s", ip))
				if err != nil {
					panic(err)
				}

				if !containsAddr(raddrs, pub) {
					raddrs = append(raddrs, pub)
				}
				continue
			}

			ip, err = addr.ValueForProtocol(ma.P_IP6)
			if err == nil {
				pub, err := ma.NewMultiaddr(fmt.Sprintf("/ip6/%s", ip))
				if err != nil {
					panic(err)
				}
				if !containsAddr(raddrs, pub) {
					raddrs = append(raddrs, pub)
				}
				continue
			}
		} else {
			raddrs = append(raddrs, addr)
		}
	}

	for _, pi := range h.relays {
		circuit, err := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s/p2p-circuit", pi.ID.Pretty()))
		if err != nil {
			panic(err)
		}

		for _, addr := range pi.Addrs {
			if !manet.IsPrivateAddr(addr) {
				pub := addr.Encapsulate(circuit)
				raddrs = append(raddrs, pub)
			}
		}
	}

	h.addrs = raddrs
}

func shuffleRelays(pis []pstore.PeerInfo) {
	for i := range pis {
		j := rand.Intn(i + 1)
		pis[i], pis[j] = pis[j], pis[i]
	}
}

func containsAddr(lst []ma.Multiaddr, addr ma.Multiaddr) bool {
	for _, xaddr := range lst {
		if xaddr.Equal(addr) {
			return true
		}
	}
	return false
}

// notify
func (h *AutoRelayHost) Listen(inet.Network, ma.Multiaddr)      {}
func (h *AutoRelayHost) ListenClose(inet.Network, ma.Multiaddr) {}
func (h *AutoRelayHost) Connected(inet.Network, inet.Conn)      {}

func (h *AutoRelayHost) Disconnected(_ inet.Network, c inet.Conn) {
	p := c.RemotePeer()
	h.mx.Lock()
	defer h.mx.Unlock()
	if _, ok := h.relays[p]; ok {
		delete(h.relays, p)
		select {
		case h.disconnect <- struct{}{}:
		default:
		}
	}
}

func (h *AutoRelayHost) OpenedStream(inet.Network, inet.Stream) {}
func (h *AutoRelayHost) ClosedStream(inet.Network, inet.Stream) {}

var _ host.Host = (*AutoRelayHost)(nil)
<commit_msg>reduce boot delay to 60s<commit_after>package relay

import (
	"context"
	"fmt"
	"math/rand"
	"sync"
	"time"

	basic "github.com/libp2p/go-libp2p/p2p/host/basic"

	autonat "github.com/libp2p/go-libp2p-autonat"
	discovery "github.com/libp2p/go-libp2p-discovery"
	host "github.com/libp2p/go-libp2p-host"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

var (
	DesiredRelays = 3

	BootDelay = 60 * time.Second
)

// AutoRelayHost is a Host that uses relays for connectivity when a NAT is detected.
type AutoRelayHost struct {
	*basic.BasicHost
	discover discovery.Discoverer
	autonat  autonat.AutoNAT
	addrsF   basic.AddrsFactory

	disconnect chan struct{}

	mx     sync.Mutex
	relays map[peer.ID]pstore.PeerInfo
	addrs  []ma.Multiaddr
}

func NewAutoRelayHost(ctx context.Context, bhost *basic.BasicHost, discover discovery.Discoverer) *AutoRelayHost {
	autonat := autonat.NewAutoNAT(ctx, bhost, bhost.AllAddrs)
	h := &AutoRelayHost{
		BasicHost:  bhost,
		discover:   discover,
		autonat:    autonat,
		addrsF:     bhost.AddrsFactory,
		relays:     make(map[peer.ID]pstore.PeerInfo),
		disconnect: make(chan struct{}, 1),
	}
	bhost.AddrsFactory = h.hostAddrs
	bhost.Network().Notify(h)
	go h.background(ctx)
	return h
}

func (h *AutoRelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr {
	h.mx.Lock()
	defer h.mx.Unlock()
	if h.addrs != nil && h.autonat.Status() == autonat.NATStatusPrivate {
		return h.addrs
	} else {
		return h.addrsF(addrs)
	}
}

func (h *AutoRelayHost) background(ctx context.Context) {
	select {
	case <-time.After(autonat.AutoNATBootDelay + BootDelay):
	case <-ctx.Done():
		return
	}

	for {
		wait := autonat.AutoNATRefreshInterval
		switch h.autonat.Status() {
		case autonat.NATStatusUnknown:
			wait = autonat.AutoNATRetryInterval
		case autonat.NATStatusPublic:
		case autonat.NATStatusPrivate:
			h.findRelays(ctx)
		}

		select {
		case <-h.disconnect:
			// invalidate addrs
			h.mx.Lock()
			h.addrs = nil
			h.mx.Unlock()
		case <-time.After(wait):
		case <-ctx.Done():
			return
		}
	}
}

func (h *AutoRelayHost) findRelays(ctx context.Context) {
	h.mx.Lock()
	if len(h.relays) >= DesiredRelays {
		h.mx.Unlock()
		return
	}
	need := DesiredRelays - len(h.relays)
	h.mx.Unlock()

	limit := 20
	for ; need > limit; limit *= 2 {
	}

	dctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	pis, err := discovery.FindPeers(dctx, h.discover, "/libp2p/relay", limit)
	cancel()
	if err != nil {
		log.Debugf("error discovering relays: %s", err.Error())
		return
	}

	// TODO better relay selection strategy; this just selects random relays
	//      but we should probably use ping latency as the selection metric
	shuffleRelays(pis)

	update := 0

	for _, pi := range pis {
		h.mx.Lock()
		if _, ok := h.relays[pi.ID]; ok {
			h.mx.Unlock()
			continue
		}
		h.mx.Unlock()

		cctx, cancel := context.WithTimeout(ctx, 60*time.Second)
		err = h.Connect(cctx, pi)
		cancel()
		if err != nil {
			log.Debugf("error connecting to relay %s: %s", pi.ID, err.Error())
			continue
		}

		log.Debugf("connected to relay %s", pi.ID)
		h.mx.Lock()
		h.relays[pi.ID] = pi
		h.mx.Unlock()

		update++
		need--
		if need == 0 {
			break
		}
	}

	if update > 0 || h.addrs == nil {
		h.updateAddrs()
	}
}

func (h *AutoRelayHost) updateAddrs() {
	h.doUpdateAddrs()
	h.PushIdentify()
}

func (h *AutoRelayHost) doUpdateAddrs() {
	h.mx.Lock()
	defer h.mx.Unlock()

	addrs := h.addrsF(h.AllAddrs())
	raddrs := make([]ma.Multiaddr, 0, len(addrs)+len(h.relays))

	// remove our public addresses from the list and replace them by just the public IP
	for _, addr := range addrs {
		if manet.IsPublicAddr(addr) {
			ip, err := addr.ValueForProtocol(ma.P_IP4)
			if err == nil {
				pub, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s", ip))
				if err != nil {
					panic(err)
				}

				if !containsAddr(raddrs, pub) {
					raddrs = append(raddrs, pub)
				}
				continue
			}

			ip, err = addr.ValueForProtocol(ma.P_IP6)
			if err == nil {
				pub, err := ma.NewMultiaddr(fmt.Sprintf("/ip6/%s", ip))
				if err != nil {
					panic(err)
				}
				if !containsAddr(raddrs, pub) {
					raddrs = append(raddrs, pub)
				}
				continue
			}
		} else {
			raddrs = append(raddrs, addr)
		}
	}

	for _, pi := range h.relays {
		circuit, err := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s/p2p-circuit", pi.ID.Pretty()))
		if err != nil {
			panic(err)
		}

		for _, addr := range pi.Addrs {
			if !manet.IsPrivateAddr(addr) {
				pub := addr.Encapsulate(circuit)
				raddrs = append(raddrs, pub)
			}
		}
	}

	h.addrs = raddrs
}

func shuffleRelays(pis []pstore.PeerInfo) {
	for i := range pis {
		j := rand.Intn(i + 1)
		pis[i], pis[j] = pis[j], pis[i]
	}
}

func containsAddr(lst []ma.Multiaddr, addr ma.Multiaddr) bool {
	for _, xaddr := range lst {
		if xaddr.Equal(addr) {
			return true
		}
	}
	return false
}

// notify
func (h *AutoRelayHost) Listen(inet.Network, ma.Multiaddr)      {}
func (h *AutoRelayHost) ListenClose(inet.Network, ma.Multiaddr) {}
func (h *AutoRelayHost) Connected(inet.Network, inet.Conn)      {}

func (h *AutoRelayHost) Disconnected(_ inet.Network, c inet.Conn) {
	p := c.RemotePeer()
	h.mx.Lock()
	defer h.mx.Unlock()
	if _, ok := h.relays[p]; ok {
		delete(h.relays, p)
		select {
		case h.disconnect <- struct{}{}:
		default:
		}
	}
}

func (h *AutoRelayHost) OpenedStream(inet.Network, inet.Stream) {}
func (h *AutoRelayHost) ClosedStream(inet.Network, inet.Stream) {}

var _ host.Host = (*AutoRelayHost)(nil)
<|endoftext|>


<commit_before>package swarm_test

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net"
	"sync"
	"testing"
	"time"

	logging "github.com/ipfs/go-log"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	ma "github.com/multiformats/go-multiaddr"

	. "github.com/libp2p/go-libp2p-swarm"
	. "github.com/libp2p/go-libp2p-swarm/testing"
)

var log = logging.Logger("swarm_test")

func EchoStreamHandler(stream inet.Stream) {
	go func() {
		defer stream.Close()

		// pull out the ipfs conn
		c := stream.Conn()
		log.Infof("%s ponging to %s", c.LocalPeer(), c.RemotePeer())

		buf := make([]byte, 4)

		for {
			if _, err := stream.Read(buf); err != nil {
				if err != io.EOF {
					log.Error("ping receive error:", err)
				}
				return
			}

			if !bytes.Equal(buf, []byte("ping")) {
				log.Errorf("ping receive error: ping != %s %v", buf, buf)
				return
			}

			if _, err := stream.Write([]byte("pong")); err != nil {
				log.Error("pond send error:", err)
				return
			}
		}

	}()
}

func makeDialOnlySwarm(ctx context.Context, t *testing.T) *Swarm {
	swarm := GenSwarm(t, ctx, OptDialOnly)
	swarm.SetStreamHandler(EchoStreamHandler)

	return swarm
}

func makeSwarms(ctx context.Context, t *testing.T, num int, opts ...Option) []*Swarm {
	swarms := make([]*Swarm, 0, num)

	for i := 0; i < num; i++ {
		swarm := GenSwarm(t, ctx, opts...)
		swarm.SetStreamHandler(EchoStreamHandler)
		swarms = append(swarms, swarm)
	}

	return swarms
}

func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) {

	var wg sync.WaitGroup
	connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) {
		// TODO: make a DialAddr func.
		s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL)
		if _, err := s.DialPeer(ctx, dst); err != nil {
			t.Fatal("error swarm dialing to peer", err)
		}
		wg.Done()
	}

	log.Info("Connecting swarms simultaneously.")
	for i, s1 := range swarms {
		for _, s2 := range swarms[i+1:] {
			wg.Add(1)
			connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) // try the first.
		}
	}
	wg.Wait()

	for _, s := range swarms {
		log.Infof("%s swarm routing table: %s", s.LocalPeer(), s.Peers())
	}
}

func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) {
	// t.Skip("skipping for another test")

	ctx := context.Background()
	swarms := makeSwarms(ctx, t, SwarmNum, OptDisableReuseport)

	// connect everyone
	connectSwarms(t, ctx, swarms)

	// ping/pong
	for _, s1 := range swarms {
		log.Debugf("-------------------------------------------------------")
		log.Debugf("%s ping pong round", s1.LocalPeer())
		log.Debugf("-------------------------------------------------------")

		_, cancel := context.WithCancel(ctx)
		got := map[peer.ID]int{}
		errChan := make(chan error, MsgNum*len(swarms))
		streamChan := make(chan inet.Stream, MsgNum)

		// send out "ping" x MsgNum to every peer
		go func() {
			defer close(streamChan)

			var wg sync.WaitGroup
			send := func(p peer.ID) {
				defer wg.Done()

				// first, one stream per peer (nice)
				stream, err := s1.NewStream(ctx, p)
				if err != nil {
					errChan <- err
					return
				}

				// send out ping!
				for k := 0; k < MsgNum; k++ { // with k messages
					msg := "ping"
					log.Debugf("%s %s %s (%d)", s1.LocalPeer(), msg, p, k)
					if _, err := stream.Write([]byte(msg)); err != nil {
						errChan <- err
						continue
					}
				}

				// read it later
				streamChan <- stream
			}

			for _, s2 := range swarms {
				if s2.LocalPeer() == s1.LocalPeer() {
					continue // dont send to self...
				}

				wg.Add(1)
				go send(s2.LocalPeer())
			}
			wg.Wait()
		}()

		// receive "pong" x MsgNum from every peer
		go func() {
			defer close(errChan)
			count := 0
			countShouldBe := MsgNum * (len(swarms) - 1)
			for stream := range streamChan { // one per peer
				defer stream.Close()

				// get peer on the other side
				p := stream.Conn().RemotePeer()

				// receive pings
				msgCount := 0
				msg := make([]byte, 4)
				for k := 0; k < MsgNum; k++ { // with k messages

					// read from the stream
					if _, err := stream.Read(msg); err != nil {
						errChan <- err
						continue
					}

					if string(msg) != "pong" {
						errChan <- fmt.Errorf("unexpected message: %s", msg)
						continue
					}

					log.Debugf("%s %s %s (%d)", s1.LocalPeer(), msg, p, k)
					msgCount++
				}

				got[p] = msgCount
				count += msgCount
			}

			if count != countShouldBe {
				errChan <- fmt.Errorf("count mismatch: %d != %d", count, countShouldBe)
			}
		}()

		// check any errors (blocks till consumer is done)
		for err := range errChan {
			if err != nil {
				t.Error(err.Error())
			}
		}

		log.Debugf("%s got pongs", s1.LocalPeer())
		if (len(swarms) - 1) != len(got) {
			t.Errorf("got (%d) less messages than sent (%d).", len(got), len(swarms))
		}

		for p, n := range got {
			if n != MsgNum {
				t.Error("peer did not get all msgs", p, n, "/", MsgNum)
			}
		}

		cancel()
		<-time.After(10 * time.Millisecond)
	}

	for _, s := range swarms {
		s.Close()
	}
}

func TestSwarm(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	// msgs := 1000
	msgs := 100
	swarms := 5
	SubtestSwarm(t, swarms, msgs)
}

func TestBasicSwarm(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	msgs := 1
	swarms := 2
	SubtestSwarm(t, swarms, msgs)
}

func TestConnHandler(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 5)

	gotconn := make(chan struct{}, 10)
	swarms[0].SetConnHandler(func(conn inet.Conn) {
		gotconn <- struct{}{}
	})

	connectSwarms(t, ctx, swarms)

	<-time.After(time.Millisecond)
	// should've gotten 5 by now.

	swarms[0].SetConnHandler(nil)

	expect := 4
	for i := 0; i < expect; i++ {
		select {
		case <-time.After(time.Second):
			t.Fatal("failed to get connections")
		case <-gotconn:
		}
	}

	select {
	case <-gotconn:
		t.Fatalf("should have connected to %d swarms, got an extra.", expect)
	default:
	}
}

func TestAddrBlocking(t *testing.T) {
	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	swarms[0].SetConnHandler(func(conn inet.Conn) {
		t.Errorf("no connections should happen! -- %s", conn)
	})

	_, block, err := net.ParseCIDR("127.0.0.1/8")
	if err != nil {
		t.Fatal(err)
	}

	swarms[1].Filters.AddDialFilter(block)

	swarms[1].Peerstore().AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], pstore.PermanentAddrTTL)
	_, err = swarms[1].DialPeer(ctx, swarms[0].LocalPeer())
	if err == nil {
		t.Fatal("dial should have failed")
	}

	swarms[0].Peerstore().AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], pstore.PermanentAddrTTL)
	_, err = swarms[0].DialPeer(ctx, swarms[1].LocalPeer())
	if err == nil {
		t.Fatal("dial should have failed")
	}
}

func TestFilterBounds(t *testing.T) {
	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	conns := make(chan struct{}, 8)
	swarms[0].SetConnHandler(func(conn inet.Conn) {
		conns <- struct{}{}
	})

	// Address that we wont be dialing from
	_, block, err := net.ParseCIDR("192.0.0.1/8")
	if err != nil {
		t.Fatal(err)
	}

	// set filter on both sides, shouldnt matter
	swarms[1].Filters.AddDialFilter(block)
	swarms[0].Filters.AddDialFilter(block)

	connectSwarms(t, ctx, swarms)

	select {
	case <-time.After(time.Second):
		t.Fatal("should have gotten connection")
	case <-conns:
		t.Log("got connect")
	}
}

func TestNoDial(t *testing.T) {
	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	_, err := swarms[0].NewStream(inet.WithNoDial(ctx, "swarm test"), swarms[1].LocalPeer())
	if err != inet.ErrNoConn {
		t.Fatal("should have failed with ErrNoConn")
	}
}
<commit_msg>kill stray line<commit_after>package swarm_test

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net"
	"sync"
	"testing"
	"time"

	logging "github.com/ipfs/go-log"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	ma "github.com/multiformats/go-multiaddr"

	. "github.com/libp2p/go-libp2p-swarm"
	. "github.com/libp2p/go-libp2p-swarm/testing"
)

var log = logging.Logger("swarm_test")

func EchoStreamHandler(stream inet.Stream) {
	go func() {
		defer stream.Close()

		// pull out the ipfs conn
		c := stream.Conn()
		log.Infof("%s ponging to %s", c.LocalPeer(), c.RemotePeer())

		buf := make([]byte, 4)

		for {
			if _, err := stream.Read(buf); err != nil {
				if err != io.EOF {
					log.Error("ping receive error:", err)
				}
				return
			}

			if !bytes.Equal(buf, []byte("ping")) {
				log.Errorf("ping receive error: ping != %s %v", buf, buf)
				return
			}

			if _, err := stream.Write([]byte("pong")); err != nil {
				log.Error("pond send error:", err)
				return
			}
		}
	}()
}

func makeDialOnlySwarm(ctx context.Context, t *testing.T) *Swarm {
	swarm := GenSwarm(t, ctx, OptDialOnly)
	swarm.SetStreamHandler(EchoStreamHandler)

	return swarm
}

func makeSwarms(ctx context.Context, t *testing.T, num int, opts ...Option) []*Swarm {
	swarms := make([]*Swarm, 0, num)

	for i := 0; i < num; i++ {
		swarm := GenSwarm(t, ctx, opts...)
		swarm.SetStreamHandler(EchoStreamHandler)
		swarms = append(swarms, swarm)
	}

	return swarms
}

func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) {

	var wg sync.WaitGroup
	connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) {
		// TODO: make a DialAddr func.
		s.Peerstore().AddAddr(dst, addr, pstore.PermanentAddrTTL)
		if _, err := s.DialPeer(ctx, dst); err != nil {
			t.Fatal("error swarm dialing to peer", err)
		}
		wg.Done()
	}

	log.Info("Connecting swarms simultaneously.")
	for i, s1 := range swarms {
		for _, s2 := range swarms[i+1:] {
			wg.Add(1)
			connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) // try the first.
		}
	}
	wg.Wait()

	for _, s := range swarms {
		log.Infof("%s swarm routing table: %s", s.LocalPeer(), s.Peers())
	}
}

func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) {
	// t.Skip("skipping for another test")

	ctx := context.Background()
	swarms := makeSwarms(ctx, t, SwarmNum, OptDisableReuseport)

	// connect everyone
	connectSwarms(t, ctx, swarms)

	// ping/pong
	for _, s1 := range swarms {
		log.Debugf("-------------------------------------------------------")
		log.Debugf("%s ping pong round", s1.LocalPeer())
		log.Debugf("-------------------------------------------------------")

		_, cancel := context.WithCancel(ctx)
		got := map[peer.ID]int{}
		errChan := make(chan error, MsgNum*len(swarms))
		streamChan := make(chan inet.Stream, MsgNum)

		// send out "ping" x MsgNum to every peer
		go func() {
			defer close(streamChan)

			var wg sync.WaitGroup
			send := func(p peer.ID) {
				defer wg.Done()

				// first, one stream per peer (nice)
				stream, err := s1.NewStream(ctx, p)
				if err != nil {
					errChan <- err
					return
				}

				// send out ping!
				for k := 0; k < MsgNum; k++ { // with k messages
					msg := "ping"
					log.Debugf("%s %s %s (%d)", s1.LocalPeer(), msg, p, k)
					if _, err := stream.Write([]byte(msg)); err != nil {
						errChan <- err
						continue
					}
				}

				// read it later
				streamChan <- stream
			}

			for _, s2 := range swarms {
				if s2.LocalPeer() == s1.LocalPeer() {
					continue // dont send to self...
				}

				wg.Add(1)
				go send(s2.LocalPeer())
			}
			wg.Wait()
		}()

		// receive "pong" x MsgNum from every peer
		go func() {
			defer close(errChan)
			count := 0
			countShouldBe := MsgNum * (len(swarms) - 1)
			for stream := range streamChan { // one per peer
				defer stream.Close()

				// get peer on the other side
				p := stream.Conn().RemotePeer()

				// receive pings
				msgCount := 0
				msg := make([]byte, 4)
				for k := 0; k < MsgNum; k++ { // with k messages

					// read from the stream
					if _, err := stream.Read(msg); err != nil {
						errChan <- err
						continue
					}

					if string(msg) != "pong" {
						errChan <- fmt.Errorf("unexpected message: %s", msg)
						continue
					}

					log.Debugf("%s %s %s (%d)", s1.LocalPeer(), msg, p, k)
					msgCount++
				}

				got[p] = msgCount
				count += msgCount
			}

			if count != countShouldBe {
				errChan <- fmt.Errorf("count mismatch: %d != %d", count, countShouldBe)
			}
		}()

		// check any errors (blocks till consumer is done)
		for err := range errChan {
			if err != nil {
				t.Error(err.Error())
			}
		}

		log.Debugf("%s got pongs", s1.LocalPeer())
		if (len(swarms) - 1) != len(got) {
			t.Errorf("got (%d) less messages than sent (%d).", len(got), len(swarms))
		}

		for p, n := range got {
			if n != MsgNum {
				t.Error("peer did not get all msgs", p, n, "/", MsgNum)
			}
		}

		cancel()
		<-time.After(10 * time.Millisecond)
	}

	for _, s := range swarms {
		s.Close()
	}
}

func TestSwarm(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	// msgs := 1000
	msgs := 100
	swarms := 5
	SubtestSwarm(t, swarms, msgs)
}

func TestBasicSwarm(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	msgs := 1
	swarms := 2
	SubtestSwarm(t, swarms, msgs)
}

func TestConnHandler(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 5)

	gotconn := make(chan struct{}, 10)
	swarms[0].SetConnHandler(func(conn inet.Conn) {
		gotconn <- struct{}{}
	})

	connectSwarms(t, ctx, swarms)

	<-time.After(time.Millisecond)
	// should've gotten 5 by now.

	swarms[0].SetConnHandler(nil)

	expect := 4
	for i := 0; i < expect; i++ {
		select {
		case <-time.After(time.Second):
			t.Fatal("failed to get connections")
		case <-gotconn:
		}
	}

	select {
	case <-gotconn:
		t.Fatalf("should have connected to %d swarms, got an extra.", expect)
	default:
	}
}

func TestAddrBlocking(t *testing.T) {
	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	swarms[0].SetConnHandler(func(conn inet.Conn) {
		t.Errorf("no connections should happen! -- %s", conn)
	})

	_, block, err := net.ParseCIDR("127.0.0.1/8")
	if err != nil {
		t.Fatal(err)
	}

	swarms[1].Filters.AddDialFilter(block)

	swarms[1].Peerstore().AddAddr(swarms[0].LocalPeer(), swarms[0].ListenAddresses()[0], pstore.PermanentAddrTTL)
	_, err = swarms[1].DialPeer(ctx, swarms[0].LocalPeer())
	if err == nil {
		t.Fatal("dial should have failed")
	}

	swarms[0].Peerstore().AddAddr(swarms[1].LocalPeer(), swarms[1].ListenAddresses()[0], pstore.PermanentAddrTTL)
	_, err = swarms[0].DialPeer(ctx, swarms[1].LocalPeer())
	if err == nil {
		t.Fatal("dial should have failed")
	}
}

func TestFilterBounds(t *testing.T) {
	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	conns := make(chan struct{}, 8)
	swarms[0].SetConnHandler(func(conn inet.Conn) {
		conns <- struct{}{}
	})

	// Address that we wont be dialing from
	_, block, err := net.ParseCIDR("192.0.0.1/8")
	if err != nil {
		t.Fatal(err)
	}

	// set filter on both sides, shouldnt matter
	swarms[1].Filters.AddDialFilter(block)
	swarms[0].Filters.AddDialFilter(block)

	connectSwarms(t, ctx, swarms)

	select {
	case <-time.After(time.Second):
		t.Fatal("should have gotten connection")
	case <-conns:
		t.Log("got connect")
	}
}

func TestNoDial(t *testing.T) {
	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	_, err := swarms[0].NewStream(inet.WithNoDial(ctx, "swarm test"), swarms[1].LocalPeer())
	if err != inet.ErrNoConn {
		t.Fatal("should have failed with ErrNoConn")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 The Decred developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
)

// MiningErrorCode identifies a kind of error.
type MiningErrorCode int

// These constants are used to identify a specific RuleError.
const (
	// ErrNotEnoughVoters indicates that there were not enough voters to
	// build a block on top of HEAD.
	ErrNotEnoughVoters MiningErrorCode = iota

	// ErrFailedToGetGeneration specifies that the current generation for
	// a block could not be obtained from blockchain.
	ErrFailedToGetGeneration

	// ErrGetStakeDifficulty indicates that the current stake difficulty
	// could not be obtained.
	ErrGetStakeDifficulty

	// ErrGetStakeDifficulty indicates that the current top block of the
	// blockchain could not be obtained.
	ErrGetTopBlock

	// ErrCreatingCoinbase indicates that there was a problem generating
	// the coinbase.
	ErrCreatingCoinbase

	// ErrGettingMedianTime indicates that the server was unable to get the
	// median adjusted time for the network.
	ErrGettingMedianTime

	// ErrGettingDifficulty indicates that there was an error getting the
	// PoW difficulty.
	ErrGettingDifficulty

	// ErrTransactionAppend indicates there was a problem adding a msgtx
	// to a msgblock.
	ErrTransactionAppend

	// ErrCheckConnectBlock indicates that a newly created block template
	// failed blockchain.CheckBlockSanity.
	ErrCheckBlockSanity

	// ErrCheckConnectBlock indicates that a newly created block template
	// failed blockchain.CheckConnectBlock.
	ErrCheckConnectBlock

	// ErrCoinbaseLengthOverflow indicates that a coinbase length was overflowed,
	// probably of a result of incrementing extranonce.
	ErrCoinbaseLengthOverflow

	// ErrFraudProofIndex indicates that there was an error finding the index
	// for a fraud proof.
	ErrFraudProofIndex

	// ErrFetchTxStore indicates a transaction store failed to fetch.
	ErrFetchTxStore
)

// Map of MiningErrorCode values back to their constant names for pretty printing.
var miningErrorCodeStrings = map[MiningErrorCode]string{
	ErrNotEnoughVoters:        "ErrNotEnoughVoters",
	ErrFailedToGetGeneration:  "ErrFailedToGetGeneration",
	ErrGetStakeDifficulty:     "ErrGetStakeDifficulty",
	ErrGetTopBlock:            "ErrGetTopBlock",
	ErrCreatingCoinbase:       "ErrCreatingCoinbase",
	ErrGettingMedianTime:      "ErrGettingMedianTime",
	ErrGettingDifficulty:      "ErrGettingDifficulty",
	ErrTransactionAppend:      "ErrTransactionAppend",
	ErrCheckBlockSanity:       "ErrCheckBlockSanity",
	ErrCheckConnectBlock:      "ErrCheckConnectBlock",
	ErrCoinbaseLengthOverflow: "ErrCoinbaseLengthOverflow",
	ErrFraudProofIndex:        "ErrFraudProofIndex",
	ErrFetchTxStore:           "ErrFetchTxStore",
}

// String returns the MiningErrorCode as a human-readable name.
func (e MiningErrorCode) String() string {
	if s := miningErrorCodeStrings[e]; s != "" {
		return s
	}
	return fmt.Sprintf("Unknown MiningErrorCode (%d)", int(e))
}

// MiningRuleError identifies a rule violation.  It is used to indicate that
// processing of a block or transaction failed due to one of the many validation
// rules.  The caller can use type assertions to determine if a failure was
// specifically due to a rule violation and access the MiningErrorCode field to
// ascertain the specific reason for the rule violation.
type MiningRuleError struct {
	ErrorCode   MiningErrorCode // Describes the kind of error
	Description string          // Human readable description of the issue
}

// Error satisfies the error interface and prints human-readable errors.
func (e MiningRuleError) Error() string {
	return e.Description
}

// GetCode satisfies the error interface and prints human-readable errors.
func (e MiningRuleError) GetCode() MiningErrorCode {
	return e.ErrorCode
}

// miningRuleError creates an RuleError given a set of arguments.
func miningRuleError(c MiningErrorCode, desc string) MiningRuleError {
	return MiningRuleError{ErrorCode: c, Description: desc}
}
<commit_msg>miningerror: Remove duplicate copyright.<commit_after>// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
)

// MiningErrorCode identifies a kind of error.
type MiningErrorCode int

// These constants are used to identify a specific RuleError.
const (
	// ErrNotEnoughVoters indicates that there were not enough voters to
	// build a block on top of HEAD.
	ErrNotEnoughVoters MiningErrorCode = iota

	// ErrFailedToGetGeneration specifies that the current generation for
	// a block could not be obtained from blockchain.
	ErrFailedToGetGeneration

	// ErrGetStakeDifficulty indicates that the current stake difficulty
	// could not be obtained.
	ErrGetStakeDifficulty

	// ErrGetStakeDifficulty indicates that the current top block of the
	// blockchain could not be obtained.
	ErrGetTopBlock

	// ErrCreatingCoinbase indicates that there was a problem generating
	// the coinbase.
	ErrCreatingCoinbase

	// ErrGettingMedianTime indicates that the server was unable to get the
	// median adjusted time for the network.
	ErrGettingMedianTime

	// ErrGettingDifficulty indicates that there was an error getting the
	// PoW difficulty.
	ErrGettingDifficulty

	// ErrTransactionAppend indicates there was a problem adding a msgtx
	// to a msgblock.
	ErrTransactionAppend

	// ErrCheckConnectBlock indicates that a newly created block template
	// failed blockchain.CheckBlockSanity.
	ErrCheckBlockSanity

	// ErrCheckConnectBlock indicates that a newly created block template
	// failed blockchain.CheckConnectBlock.
	ErrCheckConnectBlock

	// ErrCoinbaseLengthOverflow indicates that a coinbase length was overflowed,
	// probably of a result of incrementing extranonce.
	ErrCoinbaseLengthOverflow

	// ErrFraudProofIndex indicates that there was an error finding the index
	// for a fraud proof.
	ErrFraudProofIndex

	// ErrFetchTxStore indicates a transaction store failed to fetch.
	ErrFetchTxStore
)

// Map of MiningErrorCode values back to their constant names for pretty printing.
var miningErrorCodeStrings = map[MiningErrorCode]string{
	ErrNotEnoughVoters:        "ErrNotEnoughVoters",
	ErrFailedToGetGeneration:  "ErrFailedToGetGeneration",
	ErrGetStakeDifficulty:     "ErrGetStakeDifficulty",
	ErrGetTopBlock:            "ErrGetTopBlock",
	ErrCreatingCoinbase:       "ErrCreatingCoinbase",
	ErrGettingMedianTime:      "ErrGettingMedianTime",
	ErrGettingDifficulty:      "ErrGettingDifficulty",
	ErrTransactionAppend:      "ErrTransactionAppend",
	ErrCheckBlockSanity:       "ErrCheckBlockSanity",
	ErrCheckConnectBlock:      "ErrCheckConnectBlock",
	ErrCoinbaseLengthOverflow: "ErrCoinbaseLengthOverflow",
	ErrFraudProofIndex:        "ErrFraudProofIndex",
	ErrFetchTxStore:           "ErrFetchTxStore",
}

// String returns the MiningErrorCode as a human-readable name.
func (e MiningErrorCode) String() string {
	if s := miningErrorCodeStrings[e]; s != "" {
		return s
	}
	return fmt.Sprintf("Unknown MiningErrorCode (%d)", int(e))
}

// MiningRuleError identifies a rule violation.  It is used to indicate that
// processing of a block or transaction failed due to one of the many validation
// rules.  The caller can use type assertions to determine if a failure was
// specifically due to a rule violation and access the MiningErrorCode field to
// ascertain the specific reason for the rule violation.
type MiningRuleError struct {
	ErrorCode   MiningErrorCode // Describes the kind of error
	Description string          // Human readable description of the issue
}

// Error satisfies the error interface and prints human-readable errors.
func (e MiningRuleError) Error() string {
	return e.Description
}

// GetCode satisfies the error interface and prints human-readable errors.
func (e MiningRuleError) GetCode() MiningErrorCode {
	return e.ErrorCode
}

// miningRuleError creates an RuleError given a set of arguments.
func miningRuleError(c MiningErrorCode, desc string) MiningRuleError {
	return MiningRuleError{ErrorCode: c, Description: desc}
}
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strings"
)

const (
	slackEventTypeMessage = "message"

	maxMessageSize  = 4000
	maxMessageLines = 25
)

// Processor type processes inbound events from Slack
type Processor struct {
	// Connection to Slack
	con *Connection

	// Slack user information relating to the bot account
	self User

	// a sequence number to uniquely identify sent messages and correlate with acks from Slack
	sequence int

	// map of event handler functions to handle types of Slack event
	eventHandlers map[string]func(*Processor, map[string]interface{}, []byte)

	// map of users who are members of the Slack group
	users map[string]User
}

// event type represents an event sent to Slack e.g. messages
type event struct {
	Id      int    `json:"id"`
	Type    string `json:"type"`
	Channel string `json:"channel"`
	Text    string `json:"text"`
}

// user_change event type represents a user change event from Slack
type userChangeEvent struct {
	Type        string `json:"type"`
	UpdatedUser User   `json:"user"`
}

// send Event to Slack
func (p *Processor) sendEvent(eventType string, channel string, text string) error {
	p.sequence++

	response := &event{Id: p.sequence, Type: eventType, Channel: channel, Text: text}

	responseJson, err := json.Marshal(response)
	if err != nil {
		return err
	}

	p.con.Write(responseJson)

	return nil
}

/*
// Write the message on the specified channel to Slack
func (p *Processor) Write(channel string, text string) error {
	for len(text) > 0 {
		if len(text) <= maxMessageSize {
			if err := p.sendEvent(slackEventTypeMessage, channel, text); err != nil {
				return err
			}
			text = ""
		} else {
			// split message at a convenient place
			maxSizeChunk := text[:maxMessageSize]

			var breakIndex int
			if lastLineBreak := strings.LastIndex(maxSizeChunk, "\n"); lastLineBreak > -1 {
				breakIndex = lastLineBreak
			} else if lastWordBreak := strings.LastIndexAny(maxSizeChunk, "\n\t .,/\\-(){}[]|=+*&"); lastWordBreak > -1 {
				breakIndex = lastWordBreak
			} else {
				breakIndex = maxMessageSize
			}

			if err := p.sendEvent(slackEventTypeMessage, channel, text[:breakIndex]); err != nil {
				return err
			}

			if breakIndex != maxMessageSize {
				breakIndex++
			}

			text = text[breakIndex:]
		}
	}

	return nil
}
*/
// Write the message on the specified channel to Slack
func (p *Processor) Write(channel string, text string) error {
	for len(text) > 0 {
		lines := strings.Count(text, "\n")
		if len(text) <= maxMessageSize && lines <= maxMessageLines {
			if err := p.sendEvent(slackEventTypeMessage, channel, text); err != nil {
				return err
			}
			text = ""
		} else {
			// split message at a convenient place
			var breakIndex int

			if len(text) > maxMessageSize {
				maxSizeChunk := text[:maxMessageSize]

				if lastLineBreak := strings.LastIndex(maxSizeChunk, "\n"); lastLineBreak > -1 {
					breakIndex = lastLineBreak
				} else if lastWordBreak := strings.LastIndexAny(maxSizeChunk, "\n\t .,/\\-(){}[]|=+*&"); lastWordBreak > -1 {
					breakIndex = lastWordBreak
				} else {
					breakIndex = maxMessageSize
				}
			} else {
				// too many lines to the message
				var index int
				for n := 0; index < len(text) && n < lines; n++ {
					p := strings.Index(text[index:], "\n")
					if p == -1 {
						break
					}
					index += p
				}
				breakIndex = index
			}

			if err := p.sendEvent(slackEventTypeMessage, channel, text[:breakIndex]); err != nil {
				return err
			}

			if breakIndex != maxMessageSize {
				breakIndex++
			}

			text = text[breakIndex:]
		}
	}

	return nil
}

// Start processing events from Slack
func (p *Processor) Start() {
	for {
		msg := p.con.Read()

		log.Printf("%s", msg)

		var data map[string]interface{}
		err := json.Unmarshal(msg, &data)

		if err != nil {
			fmt.Printf("%T\n%s\n%#v\n", err, err, err)
			switch v := err.(type) {
			case *json.SyntaxError:
				fmt.Println(string(msg[v.Offset-40 : v.Offset]))
			}
			log.Printf("%s", msg)
			continue
		}

		// if reply_to attribute is present the event is an ack' for a sent message
		_, isReply := data["reply_to"]
		subtype, ok := data["subtype"]
		var isMessageChangedEvent bool

		if ok {
			isMessageChangedEvent = (subtype.(string) == "message_changed" || subtype.(string) == "message_deleted")
		}

		if !isReply && !isMessageChangedEvent {
			handler, ok := p.eventHandlers[data["type"].(string)]

			if ok {
				handler(p, data, msg)
			}
		}
	}
}

// updateUser updates or adds (if not already existing) the specifed user
func (p *Processor) updateUser(user User) {
	p.users[user.Id] = user
}

// onConnected is a callback for when the client connects (or reconnects) to Slack.
func (p *Processor) onConnected(con *Connection) {
	p.self = con.config.Self
	log.Printf("Connected to Slack as %s", p.self.Name)

	p.users = make(map[string]User)
	for _, user := range con.config.Users {
		p.updateUser(user)
	}
}

// type for callbacks to receive messages from Slack
type messageProcessor func(*Message)

// Starts processing events on the connection from Slack and passes any messages to the hear callback and only
// messages addressed to the bot to the respond callback
func EventProcessor(con *Connection, respond messageProcessor, hear messageProcessor) {
	p := Processor{
		con:  con,
		self: con.config.Self,
		eventHandlers: map[string]func(*Processor, map[string]interface{}, []byte){
			slackEventTypeMessage: func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				filterMessage(p, event, respond, hear)
			},
			"user_change": func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				var userEvent userChangeEvent
				err := json.Unmarshal(rawEvent, &userEvent)

				if err != nil {
					fmt.Printf("%T\n%s\n%#v\n", err, err, err)
					switch v := err.(type) {
					case *json.SyntaxError:
						fmt.Println(string(rawEvent[v.Offset-40 : v.Offset]))
					}
					log.Printf("%s", rawEvent)
				}
				p.updateUser(userEvent.UpdatedUser)
			},
			"hello": func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				p.onConnected(con)
			},
			"error": func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				log.Printf("Error received from Slack: %s", rawEvent)
			},
		},
		users: make(map[string]User),
	}

	p.Start()
}

// Invoke one of the specified callbacks for the message if appropriate
func filterMessage(p *Processor, data map[string]interface{}, respond messageProcessor, hear messageProcessor) {
	var userFullName string
	var userId string

	user, ok := data["user"]
	if ok {
		userId = user.(string)
		user, exists := p.users[userId]
		if exists {
			userFullName = user.RealName
		}
	}

	// process messages directed at Talbot
	r, _ := regexp.Compile("^(<@" + p.self.Id + ">|@?" + p.self.Name + "):? (.+)")

	text, ok := data["text"]
	if !ok || text == nil {
		return
	}

	matches := r.FindStringSubmatch(text.(string))

	if len(matches) == 3 {
		if respond != nil {
			m := &Message{eventStream: p, responseStrategy: reply, Text: matches[2], From: userFullName, fromId: userId, channel: data["channel"].(string)}
			respond(m)
		}
	} else if data["channel"].(string)[0] == 'D' {
		if respond != nil {
			// process direct messages
			m := &Message{eventStream: p, responseStrategy: send, Text: text.(string), From: userFullName, fromId: userId, channel: data["channel"].(string)}
			respond(m)
		}
	} else {
		if hear != nil {
			m := &Message{eventStream: p, responseStrategy: send, Text: text.(string), From: userFullName, fromId: userId, channel: data["channel"].(string)}
			hear(m)
		}
	}
}
<commit_msg>fixed chunking<commit_after>package slack

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strings"
)

const (
	slackEventTypeMessage = "message"

	maxMessageSize  = 4000
	maxMessageLines = 25
)

// Processor type processes inbound events from Slack
type Processor struct {
	// Connection to Slack
	con *Connection

	// Slack user information relating to the bot account
	self User

	// a sequence number to uniquely identify sent messages and correlate with acks from Slack
	sequence int

	// map of event handler functions to handle types of Slack event
	eventHandlers map[string]func(*Processor, map[string]interface{}, []byte)

	// map of users who are members of the Slack group
	users map[string]User
}

// event type represents an event sent to Slack e.g. messages
type event struct {
	Id      int    `json:"id"`
	Type    string `json:"type"`
	Channel string `json:"channel"`
	Text    string `json:"text"`
}

// user_change event type represents a user change event from Slack
type userChangeEvent struct {
	Type        string `json:"type"`
	UpdatedUser User   `json:"user"`
}

// send Event to Slack
func (p *Processor) sendEvent(eventType string, channel string, text string) error {
	p.sequence++

	response := &event{Id: p.sequence, Type: eventType, Channel: channel, Text: text}

	responseJson, err := json.Marshal(response)
	if err != nil {
		return err
	}

	p.con.Write(responseJson)

	return nil
}

/*
// Write the message on the specified channel to Slack
func (p *Processor) Write(channel string, text string) error {
	for len(text) > 0 {
		if len(text) <= maxMessageSize {
			if err := p.sendEvent(slackEventTypeMessage, channel, text); err != nil {
				return err
			}
			text = ""
		} else {
			// split message at a convenient place
			maxSizeChunk := text[:maxMessageSize]

			var breakIndex int
			if lastLineBreak := strings.LastIndex(maxSizeChunk, "\n"); lastLineBreak > -1 {
				breakIndex = lastLineBreak
			} else if lastWordBreak := strings.LastIndexAny(maxSizeChunk, "\n\t .,/\\-(){}[]|=+*&"); lastWordBreak > -1 {
				breakIndex = lastWordBreak
			} else {
				breakIndex = maxMessageSize
			}

			if err := p.sendEvent(slackEventTypeMessage, channel, text[:breakIndex]); err != nil {
				return err
			}

			if breakIndex != maxMessageSize {
				breakIndex++
			}

			text = text[breakIndex:]
		}
	}

	return nil
}
*/
// Write the message on the specified channel to Slack
func (p *Processor) Write(channel string, text string) error {
	for len(text) > 0 {
		lines := strings.Count(text, "\n")
		if len(text) <= maxMessageSize && lines <= maxMessageLines {
			if err := p.sendEvent(slackEventTypeMessage, channel, text); err != nil {
				return err
			}
			text = ""
		} else {
			// split message at a convenient place
			var breakIndex int
			maxSizeChunk := text

			if len(text) > maxMessageSize {
				maxSizeChunk := text[:maxMessageSize]
				lines = strings.Count(maxSizeChunk, "\n")
			}
			if lines > maxMessageLines {
				// too many lines to the message
				var index int
				for n := 0; index < len(maxSizeChunk) && n < lines; n++ {
					p := strings.Index(maxSizeChunk[index:], "\n")
					if p == -1 {
						break
					}
					index += p
				}
				breakIndex = index
			} else if lastLineBreak := strings.LastIndex(maxSizeChunk, "\n"); lastLineBreak > -1 {
				breakIndex = lastLineBreak
			} else if lastWordBreak := strings.LastIndexAny(maxSizeChunk, "\n\t .,/\\-(){}[]|=+*&"); lastWordBreak > -1 {
				breakIndex = lastWordBreak
			} else {
				breakIndex = maxMessageSize
			}

			if err := p.sendEvent(slackEventTypeMessage, channel, text[:breakIndex]); err != nil {
				return err
			}

			if breakIndex != maxMessageSize {
				breakIndex++
			}

			text = text[breakIndex:]
		}
	}

	return nil
}

// Start processing events from Slack
func (p *Processor) Start() {
	for {
		msg := p.con.Read()

		log.Printf("%s", msg)

		var data map[string]interface{}
		err := json.Unmarshal(msg, &data)

		if err != nil {
			fmt.Printf("%T\n%s\n%#v\n", err, err, err)
			switch v := err.(type) {
			case *json.SyntaxError:
				fmt.Println(string(msg[v.Offset-40 : v.Offset]))
			}
			log.Printf("%s", msg)
			continue
		}

		// if reply_to attribute is present the event is an ack' for a sent message
		_, isReply := data["reply_to"]
		subtype, ok := data["subtype"]
		var isMessageChangedEvent bool

		if ok {
			isMessageChangedEvent = (subtype.(string) == "message_changed" || subtype.(string) == "message_deleted")
		}

		if !isReply && !isMessageChangedEvent {
			handler, ok := p.eventHandlers[data["type"].(string)]

			if ok {
				handler(p, data, msg)
			}
		}
	}
}

// updateUser updates or adds (if not already existing) the specifed user
func (p *Processor) updateUser(user User) {
	p.users[user.Id] = user
}

// onConnected is a callback for when the client connects (or reconnects) to Slack.
func (p *Processor) onConnected(con *Connection) {
	p.self = con.config.Self
	log.Printf("Connected to Slack as %s", p.self.Name)

	p.users = make(map[string]User)
	for _, user := range con.config.Users {
		p.updateUser(user)
	}
}

// type for callbacks to receive messages from Slack
type messageProcessor func(*Message)

// Starts processing events on the connection from Slack and passes any messages to the hear callback and only
// messages addressed to the bot to the respond callback
func EventProcessor(con *Connection, respond messageProcessor, hear messageProcessor) {
	p := Processor{
		con:  con,
		self: con.config.Self,
		eventHandlers: map[string]func(*Processor, map[string]interface{}, []byte){
			slackEventTypeMessage: func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				filterMessage(p, event, respond, hear)
			},
			"user_change": func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				var userEvent userChangeEvent
				err := json.Unmarshal(rawEvent, &userEvent)

				if err != nil {
					fmt.Printf("%T\n%s\n%#v\n", err, err, err)
					switch v := err.(type) {
					case *json.SyntaxError:
						fmt.Println(string(rawEvent[v.Offset-40 : v.Offset]))
					}
					log.Printf("%s", rawEvent)
				}
				p.updateUser(userEvent.UpdatedUser)
			},
			"hello": func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				p.onConnected(con)
			},
			"error": func(p *Processor, event map[string]interface{}, rawEvent []byte) {
				log.Printf("Error received from Slack: %s", rawEvent)
			},
		},
		users: make(map[string]User),
	}

	p.Start()
}

// Invoke one of the specified callbacks for the message if appropriate
func filterMessage(p *Processor, data map[string]interface{}, respond messageProcessor, hear messageProcessor) {
	var userFullName string
	var userId string

	user, ok := data["user"]
	if ok {
		userId = user.(string)
		user, exists := p.users[userId]
		if exists {
			userFullName = user.RealName
		}
	}

	// process messages directed at Talbot
	r, _ := regexp.Compile("^(<@" + p.self.Id + ">|@?" + p.self.Name + "):? (.+)")

	text, ok := data["text"]
	if !ok || text == nil {
		return
	}

	matches := r.FindStringSubmatch(text.(string))

	if len(matches) == 3 {
		if respond != nil {
			m := &Message{eventStream: p, responseStrategy: reply, Text: matches[2], From: userFullName, fromId: userId, channel: data["channel"].(string)}
			respond(m)
		}
	} else if data["channel"].(string)[0] == 'D' {
		if respond != nil {
			// process direct messages
			m := &Message{eventStream: p, responseStrategy: send, Text: text.(string), From: userFullName, fromId: userId, channel: data["channel"].(string)}
			respond(m)
		}
	} else {
		if hear != nil {
			m := &Message{eventStream: p, responseStrategy: send, Text: text.(string), From: userFullName, fromId: userId, channel: data["channel"].(string)}
			hear(m)
		}
	}
}
<|endoftext|>