<commit_before>/*
Copyright 2018 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"bytes"
	"fmt"
	"strconv"
	"time"

	"github.com/golang/glog"
	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	"github.com/openebs/maya/pkg/util"
)

// VolumeOperator is the name of the tool that makes volume-related operations.
const (
	VolumeOperator   = "iscsi"
	IstgtConfPath    = "/usr/local/etc/istgt/istgt.conf"
	IstgtStatusCmd   = "STATUS"
	IstgtRefreshCmd  = "REFRESH"
	WaitTimeForIscsi = 3 * time.Second
)

//FileOperatorVar is used for doing File Operations
var FileOperatorVar util.FileOperator

//UnixSockVar is used for communication through Unix Socket
var UnixSockVar util.UnixSock

// CreateVolume creates a new cStor volume istgt config.
func CreateVolume(cStorVolume *apis.CStorVolume) error {
	// create conf file
	text := CreateIstgtConf(cStorVolume)
	err := FileOperatorVar.Write(IstgtConfPath, text, 0644)
	if err != nil {
		glog.Errorf("Failed to write istgt.conf")
	}
	glog.Info("Done writing istgt.conf")

	// send refresh command to istgt and read the response
	_, err = UnixSockVar.SendCommand(IstgtRefreshCmd)
	if err != nil {
		glog.Info("Failed to refresh iscsi service with new configuration.")
	}
	glog.Info("Creating Iscsi Volume Successful")
	return nil

}

// CreateIstgtConf creates istgt.conf file
func CreateIstgtConf(cStorVolume *apis.CStorVolume) []byte {

	var buffer bytes.Buffer
	buffer.WriteString(`# Global section
[Global]
`)
	buffer.WriteString("  NodeBase \"" + cStorVolume.Spec.Iqn + "\"")
	buffer.WriteString(`
  PidFile "/var/run/istgt.pid"
  AuthFile "/usr/local/etc/istgt/auth.conf"
  LogFile "/usr/local/etc/istgt/logfile"
  Luworkers 1
  MediaDirectory "/mnt"
  Timeout 60
  NopInInterval 20
  MaxR2T 16
  DiscoveryAuthMethod None
  DiscoveryAuthGroup None
  MaxSessions 32
  MaxConnections 4
  FirstBurstLength 262144
  MaxBurstLength 1048576
  MaxRecvDataSegmentLength 262144
  MaxOutstandingR2T 16
  DefaultTime2Wait 2
  DefaultTime2Retain 20
  OperationalMode 0

# UnitControl section
[UnitControl]
  AuthMethod None
  AuthGroup None
`)
	buffer.WriteString("  Portal UC1 " + cStorVolume.Spec.TargetIP + ":3261\n")
	buffer.WriteString("  Netmask " + cStorVolume.Spec.TargetIP + "/8\n")
	buffer.WriteString(`
# PortalGroup section
[PortalGroup1]
`)
	buffer.WriteString("  Portal DA1 " + cStorVolume.Spec.TargetIP + ":3260\n")
	buffer.WriteString(`
# InitiatorGroup section
[InitiatorGroup1]
  InitiatorName "ALL"
  Netmask "ALL"

[InitiatorGroup2]
  InitiatorName "None"
  Netmask "None"

# LogicalUnit section
[LogicalUnit2]
`)
	buffer.WriteString("  TargetName " + cStorVolume.Name + "\n")
	buffer.WriteString("  TargetAlias nicknamefor-" + cStorVolume.Name)
	buffer.WriteString(`
  Mapping PortalGroup1 InitiatorGroup1
  AuthMethod None
  AuthGroup None
  UseDigest Auto
  ReadOnly No
`)
	buffer.WriteString("  ReplicationFactor " + strconv.Itoa(cStorVolume.Spec.ReplicationFactor) + "\n")
	buffer.WriteString("  ConsistencyFactor " + strconv.Itoa(cStorVolume.Spec.ConsistencyFactor))
	buffer.WriteString(`
  UnitType Disk
  UnitOnline Yes
  BlockLength 512
  QueueDepth 32
  Luworkers 1
`)
	buffer.WriteString("  UnitInquiry \"OpenEBS\" \"iscsi\" \"0\" \"" + string(cStorVolume.UID) + "\"")
	buffer.WriteString(`
  PhysRecordLength 4096
`)
	buffer.WriteString("  LUN0 Storage " + cStorVolume.Spec.Capacity + " 32k")
	buffer.WriteString(`
  LUN0 Option Unmap Disable
  LUN0 Option WZero Disable
  LUN0 Option ATS Disable
  LUN0 Option XCOPY Disable
`)

	return buffer.Bytes()
}

// CheckValidVolume checks for validity of CStorVolume resource.
func CheckValidVolume(cStorVolume *apis.CStorVolume) error {
	if len(string(cStorVolume.ObjectMeta.UID)) == 0 {
		return fmt.Errorf("Invalid volume resource")
	}
	if len(string(cStorVolume.Spec.TargetIP)) == 0 {
		return fmt.Errorf("targetIP cannot be empty")
	}
	if len(string(cStorVolume.Name)) == 0 {
		return fmt.Errorf("volumeName cannot be empty")
	}
	if len(string(cStorVolume.UID)) == 0 {
		return fmt.Errorf("volumeID cannot be empty")
	}
	if len(string(cStorVolume.Spec.Capacity)) == 0 {
		return fmt.Errorf("capacity cannot be empty")
	}
	if cStorVolume.Spec.ReplicationFactor == 0 {
		return fmt.Errorf("replicationFactor cannot be zero")
	}
	if cStorVolume.Spec.ConsistencyFactor == 0 {
		return fmt.Errorf("consistencyFactor cannot be zero")
	}
	if cStorVolume.Spec.ReplicationFactor < cStorVolume.Spec.ConsistencyFactor {
		return fmt.Errorf("replicationFactor cannot be less than consistencyFactor")
	}

	return nil
}

// CheckForIscsi is blocking call for checking status of istgt in cstor-istgt container.
func CheckForIscsi() {
	for {
		_, err := UnixSockVar.SendCommand(IstgtStatusCmd)
		if err != nil {
			time.Sleep(WaitTimeForIscsi)
			glog.Warningf("Waiting for istgt... err : %v", err)
			continue
		}
		break
	}
}
<commit_msg>using NodeBase from custom resource in place of NodeBase in istgt.conf (#420)<commit_after>/*
Copyright 2018 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"bytes"
	"fmt"
	"strconv"
	"time"

	"github.com/golang/glog"
	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	"github.com/openebs/maya/pkg/util"
)

// VolumeOperator is the name of the tool that makes volume-related operations.
const (
	VolumeOperator   = "iscsi"
	IstgtConfPath    = "/usr/local/etc/istgt/istgt.conf"
	IstgtStatusCmd   = "STATUS"
	IstgtRefreshCmd  = "REFRESH"
	WaitTimeForIscsi = 3 * time.Second
)

//FileOperatorVar is used for doing File Operations
var FileOperatorVar util.FileOperator

//UnixSockVar is used for communication through Unix Socket
var UnixSockVar util.UnixSock

// CreateVolume creates a new cStor volume istgt config.
func CreateVolume(cStorVolume *apis.CStorVolume) error {
	// create conf file
	text := CreateIstgtConf(cStorVolume)
	err := FileOperatorVar.Write(IstgtConfPath, text, 0644)
	if err != nil {
		glog.Errorf("Failed to write istgt.conf")
	}
	glog.Info("Done writing istgt.conf")

	// send refresh command to istgt and read the response
	_, err = UnixSockVar.SendCommand(IstgtRefreshCmd)
	if err != nil {
		glog.Info("Failed to refresh iscsi service with new configuration.")
	}
	glog.Info("Creating Iscsi Volume Successful")
	return nil

}

// CreateIstgtConf creates istgt.conf file
func CreateIstgtConf(cStorVolume *apis.CStorVolume) []byte {

	var buffer bytes.Buffer
	buffer.WriteString(`# Global section
[Global]
`)
	buffer.WriteString("  NodeBase \"" + cStorVolume.Spec.NodeBase + "\"")
	buffer.WriteString(`
  PidFile "/var/run/istgt.pid"
  AuthFile "/usr/local/etc/istgt/auth.conf"
  LogFile "/usr/local/etc/istgt/logfile"
  Luworkers 1
  MediaDirectory "/mnt"
  Timeout 60
  NopInInterval 20
  MaxR2T 16
  DiscoveryAuthMethod None
  DiscoveryAuthGroup None
  MaxSessions 32
  MaxConnections 4
  FirstBurstLength 262144
  MaxBurstLength 1048576
  MaxRecvDataSegmentLength 262144
  MaxOutstandingR2T 16
  DefaultTime2Wait 2
  DefaultTime2Retain 20
  OperationalMode 0

# UnitControl section
[UnitControl]
  AuthMethod None
  AuthGroup None
`)
	buffer.WriteString("  Portal UC1 " + cStorVolume.Spec.TargetIP + ":3261\n")
	buffer.WriteString("  Netmask " + cStorVolume.Spec.TargetIP + "/8\n")
	buffer.WriteString(`
# PortalGroup section
[PortalGroup1]
`)
	buffer.WriteString("  Portal DA1 " + cStorVolume.Spec.TargetIP + ":3260\n")
	buffer.WriteString(`
# InitiatorGroup section
[InitiatorGroup1]
  InitiatorName "ALL"
  Netmask "ALL"

[InitiatorGroup2]
  InitiatorName "None"
  Netmask "None"

# LogicalUnit section
[LogicalUnit2]
`)
	buffer.WriteString("  TargetName " + cStorVolume.Name + "\n")
	buffer.WriteString("  TargetAlias nicknamefor-" + cStorVolume.Name)
	buffer.WriteString(`
  Mapping PortalGroup1 InitiatorGroup1
  AuthMethod None
  AuthGroup None
  UseDigest Auto
  ReadOnly No
`)
	buffer.WriteString("  ReplicationFactor " + strconv.Itoa(cStorVolume.Spec.ReplicationFactor) + "\n")
	buffer.WriteString("  ConsistencyFactor " + strconv.Itoa(cStorVolume.Spec.ConsistencyFactor))
	buffer.WriteString(`
  UnitType Disk
  UnitOnline Yes
  BlockLength 512
  QueueDepth 32
  Luworkers 1
`)
	buffer.WriteString("  UnitInquiry \"OpenEBS\" \"iscsi\" \"0\" \"" + string(cStorVolume.UID) + "\"")
	buffer.WriteString(`
  PhysRecordLength 4096
`)
	buffer.WriteString("  LUN0 Storage " + cStorVolume.Spec.Capacity + " 32k")
	buffer.WriteString(`
  LUN0 Option Unmap Disable
  LUN0 Option WZero Disable
  LUN0 Option ATS Disable
  LUN0 Option XCOPY Disable
`)

	return buffer.Bytes()
}

// CheckValidVolume checks for validity of CStorVolume resource.
func CheckValidVolume(cStorVolume *apis.CStorVolume) error {
	if len(string(cStorVolume.ObjectMeta.UID)) == 0 {
		return fmt.Errorf("Invalid volume resource")
	}
	if len(string(cStorVolume.Spec.TargetIP)) == 0 {
		return fmt.Errorf("targetIP cannot be empty")
	}
	if len(string(cStorVolume.Name)) == 0 {
		return fmt.Errorf("volumeName cannot be empty")
	}
	if len(string(cStorVolume.UID)) == 0 {
		return fmt.Errorf("volumeID cannot be empty")
	}
	if len(string(cStorVolume.Spec.Capacity)) == 0 {
		return fmt.Errorf("capacity cannot be empty")
	}
	if cStorVolume.Spec.ReplicationFactor == 0 {
		return fmt.Errorf("replicationFactor cannot be zero")
	}
	if cStorVolume.Spec.ConsistencyFactor == 0 {
		return fmt.Errorf("consistencyFactor cannot be zero")
	}
	if cStorVolume.Spec.ReplicationFactor < cStorVolume.Spec.ConsistencyFactor {
		return fmt.Errorf("replicationFactor cannot be less than consistencyFactor")
	}

	return nil
}

// CheckForIscsi is blocking call for checking status of istgt in cstor-istgt container.
func CheckForIscsi() {
	for {
		_, err := UnixSockVar.SendCommand(IstgtStatusCmd)
		if err != nil {
			time.Sleep(WaitTimeForIscsi)
			glog.Warningf("Waiting for istgt... err : %v", err)
			continue
		}
		break
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	chunkSpanStr = flag.String(
		"chunkspans",
		"10min",
		"List of chunk spans separated by ':'. The 1st whisper archive gets the 1st span, 2nd the 2nd, etc",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process .wsp files",
	)
	skipUnfinishedChunks = flag.Bool(
		"skip-unfinished-chunks",
		true,
		"Chunks that have not completed their chunk span should be skipped",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	readArchivesStr = flag.String(
		"read-archives",
		"*",
		"Comma separated list of positive integers or '*' for all archives",
	)
	chunkSpans   []uint32
	readArchives map[int]struct{}
	printLock    sync.Mutex
)

func main() {
	flag.Parse()

	for _, chunkSpanStrSplit := range strings.Split(*chunkSpanStr, ",") {
		chunkSpan := dur.MustParseUNsec("chunkspan", chunkSpanStrSplit)

		if (mdata.Month_sec % chunkSpan) != 0 {
			panic("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
		}

		_, ok := chunk.RevChunkSpans[chunkSpan]
		if !ok {
			panic(fmt.Sprintf("chunkSpan %d is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", chunkSpan))
		}

		chunkSpans = append(chunkSpans, chunkSpan)
	}

	if *readArchivesStr != "*" {
		readArchives = make(map[int]struct{})
		for _, archiveIdStr := range strings.Split(*readArchivesStr, ",") {
			archiveId, err := strconv.Atoi(archiveIdStr)
			if err != nil {
				panic(fmt.Sprintf("Invalid archive id %q: %q", archiveIdStr, err))
			}
			readArchives[archiveId] = struct{}{}
		}
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	client := &http.Client{}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file %q: %q\n", file, err))
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file %q: %q\n", file, err))
			continue
		}

		log(fmt.Sprintf("Processing file %q", file))
		met, err := getMetric(w, file)
		if err != nil {
			throwError(fmt.Sprintf("Failed to get metric: %q", err))
			continue
		}

		b, err := met.MarshalCompressed()
		if err != nil {
			throwError(fmt.Sprintf("Failed to encode metric: %q", err))
			continue
		}

		req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
		if err != nil {
			panic(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		_, err = client.Do(req)
		if err != nil {
			throwError(fmt.Sprintf("Error sending request to http endpoint %q: %q", *httpEndpoint, err))
			continue
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) string {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg"
	case whisper.AggregationSum:
		return "sum"
	case whisper.AggregationMin:
		return "min"
	case whisper.AggregationMax:
		return "max"
	case whisper.AggregationLast:
		return "lst"
	default:
		return ""
	}
}

func getMetric(w *whisper.Whisper, file string) (*archive.Metric, error) {
	if len(w.Header.Archives) == 0 {
		return nil, errors.New(fmt.Sprintf("ERROR: Whisper file contains no archives: %q", file))
	}

	archives := make([]archive.Archive, 0, len(w.Header.Archives))
	var chunkSpan uint32
	var rowKey string
	var aggMethodStr string
	name := getMetricName(file)

	// md gets generated from the first archive in the whisper file
	md := getMetricData(name, int(w.Header.Archives[0].SecondsPerPoint))

	for archiveIdx, archiveInfo := range w.Header.Archives {
		if archiveIdx > 0 {

			// on the first aggregation we determin the aggregation method as string
			if aggMethodStr == "" {
				aggMethodStr = shortAggMethodString(w.Header.Metadata.AggregationMethod)

				if aggMethodStr == "" {
					return nil, errors.New(fmt.Sprintf(
						"ERROR: Aggregation method in file %s not allowed: %d(%s)\n",
						file,
						w.Header.Metadata.AggregationMethod,
						whisper.AggregationMethod(w.Header.Metadata.AggregationMethod).String(),
					))
				}
			}

			rowKey = api.AggMetricKey(
				md.Id,
				aggMethodStr,
				archiveInfo.SecondsPerPoint,
			)
		} else {
			rowKey = md.Id
		}

		// only read archive if archiveIdx is in readArchives
		if _, ok := readArchives[archiveIdx]; !ok && len(readArchives) > 0 {
			continue
		}

		encodedChunks := make([]chunk.IterGen, 0)

		if len(chunkSpans)-1 < archiveIdx {
			// if we have more archives than chunk spans are specified, we simply use the last one
			chunkSpan = chunkSpans[len(chunkSpans)-1]
		} else {
			chunkSpan = chunkSpans[archiveIdx]
		}

		points, err := w.DumpArchive(archiveIdx)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("ERROR: Failed to read archive %d in %q, skipping: %q", archiveIdx, file, err))
		}

		var point whisper.Point
		var t0, prevT0 uint32
		var c *chunk.Chunk

		for _, point = range sortPoints(points) {
			if point.Timestamp == 0 {
				continue
			}

			t0 = point.Timestamp - (point.Timestamp % chunkSpan)
			if prevT0 == 0 {
				c = chunk.New(t0)
				prevT0 = t0
			} else if prevT0 != t0 {
				log(fmt.Sprintf("Mark chunk at t0 %d as finished", prevT0))
				c.Finish()

				encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

				log(fmt.Sprintf("Create new chunk at t0 %d with chunk span %d", t0, chunkSpan))
				c = chunk.New(t0)
				prevT0 = t0
			}

			err := c.Push(point.Timestamp, point.Value)
			if err != nil {
				return nil, errors.New(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
			}
		}

		if int64(point.Timestamp) > md.Time {
			md.Time = int64(point.Timestamp)
		}

		// if the last written point was also the last one of the current chunk,
		// or if skipUnfinishedChunks is on, we close the chunk and
		if point.Timestamp == t0+chunkSpan-archiveInfo.SecondsPerPoint || !*skipUnfinishedChunks {
			log(fmt.Sprintf("Mark current (last) chunk at t0 %d as finished", t0))
			c.Finish()
			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
		}

		log(fmt.Sprintf("Whisper file %q archive %d (%q) gets %d chunks", file, archiveIdx, name, len(encodedChunks)))
		archives = append(archives, archive.Archive{
			SecondsPerPoint: archiveInfo.SecondsPerPoint,
			Points:          archiveInfo.Points,
			Chunks:          encodedChunks,
			RowKey:          rowKey,
		})
	}

	return &archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
		MetricData:        *md,
		Archives:          archives,
	}, nil
}

func getMetricData(name string, interval int) *schema.MetricData {
	md := &schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: interval,
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	return md
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>negate skipUnfinishedChunks parameter into writeUnfinishedChunks<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	chunkSpanStr = flag.String(
		"chunkspans",
		"10min",
		"List of chunk spans separated by ':'. The 1st whisper archive gets the 1st span, 2nd the 2nd, etc",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	readArchivesStr = flag.String(
		"read-archives",
		"*",
		"Comma separated list of positive integers or '*' for all archives",
	)
	chunkSpans   []uint32
	readArchives map[int]struct{}
	printLock    sync.Mutex
)

func main() {
	flag.Parse()

	for _, chunkSpanStrSplit := range strings.Split(*chunkSpanStr, ",") {
		chunkSpan := dur.MustParseUNsec("chunkspan", chunkSpanStrSplit)

		if (mdata.Month_sec % chunkSpan) != 0 {
			panic("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
		}

		_, ok := chunk.RevChunkSpans[chunkSpan]
		if !ok {
			panic(fmt.Sprintf("chunkSpan %d is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", chunkSpan))
		}

		chunkSpans = append(chunkSpans, chunkSpan)
	}

	if *readArchivesStr != "*" {
		readArchives = make(map[int]struct{})
		for _, archiveIdStr := range strings.Split(*readArchivesStr, ",") {
			archiveId, err := strconv.Atoi(archiveIdStr)
			if err != nil {
				panic(fmt.Sprintf("Invalid archive id %q: %q", archiveIdStr, err))
			}
			readArchives[archiveId] = struct{}{}
		}
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	client := &http.Client{}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file %q: %q\n", file, err))
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file %q: %q\n", file, err))
			continue
		}

		log(fmt.Sprintf("Processing file %q", file))
		met, err := getMetric(w, file)
		if err != nil {
			throwError(fmt.Sprintf("Failed to get metric: %q", err))
			continue
		}

		b, err := met.MarshalCompressed()
		if err != nil {
			throwError(fmt.Sprintf("Failed to encode metric: %q", err))
			continue
		}

		req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
		if err != nil {
			panic(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		_, err = client.Do(req)
		if err != nil {
			throwError(fmt.Sprintf("Error sending request to http endpoint %q: %q", *httpEndpoint, err))
			continue
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) string {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg"
	case whisper.AggregationSum:
		return "sum"
	case whisper.AggregationMin:
		return "min"
	case whisper.AggregationMax:
		return "max"
	case whisper.AggregationLast:
		return "lst"
	default:
		return ""
	}
}

func getMetric(w *whisper.Whisper, file string) (*archive.Metric, error) {
	if len(w.Header.Archives) == 0 {
		return nil, errors.New(fmt.Sprintf("ERROR: Whisper file contains no archives: %q", file))
	}

	archives := make([]archive.Archive, 0, len(w.Header.Archives))
	var chunkSpan uint32
	var rowKey string
	var aggMethodStr string
	name := getMetricName(file)

	// md gets generated from the first archive in the whisper file
	md := getMetricData(name, int(w.Header.Archives[0].SecondsPerPoint))

	for archiveIdx, archiveInfo := range w.Header.Archives {
		if archiveIdx > 0 {

			// on the first aggregation we determin the aggregation method as string
			if aggMethodStr == "" {
				aggMethodStr = shortAggMethodString(w.Header.Metadata.AggregationMethod)

				if aggMethodStr == "" {
					return nil, errors.New(fmt.Sprintf(
						"ERROR: Aggregation method in file %s not allowed: %d(%s)\n",
						file,
						w.Header.Metadata.AggregationMethod,
						whisper.AggregationMethod(w.Header.Metadata.AggregationMethod).String(),
					))
				}
			}

			rowKey = api.AggMetricKey(
				md.Id,
				aggMethodStr,
				archiveInfo.SecondsPerPoint,
			)
		} else {
			rowKey = md.Id
		}

		// only read archive if archiveIdx is in readArchives
		if _, ok := readArchives[archiveIdx]; !ok && len(readArchives) > 0 {
			continue
		}

		encodedChunks := make([]chunk.IterGen, 0)

		if len(chunkSpans)-1 < archiveIdx {
			// if we have more archives than chunk spans are specified, we simply use the last one
			chunkSpan = chunkSpans[len(chunkSpans)-1]
		} else {
			chunkSpan = chunkSpans[archiveIdx]
		}

		points, err := w.DumpArchive(archiveIdx)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("ERROR: Failed to read archive %d in %q, skipping: %q", archiveIdx, file, err))
		}

		var point whisper.Point
		var t0, prevT0 uint32
		var c *chunk.Chunk

		for _, point = range sortPoints(points) {
			if point.Timestamp == 0 {
				continue
			}

			t0 = point.Timestamp - (point.Timestamp % chunkSpan)
			if prevT0 == 0 {
				c = chunk.New(t0)
				prevT0 = t0
			} else if prevT0 != t0 {
				log(fmt.Sprintf("Mark chunk at t0 %d as finished", prevT0))
				c.Finish()

				encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

				log(fmt.Sprintf("Create new chunk at t0 %d with chunk span %d", t0, chunkSpan))
				c = chunk.New(t0)
				prevT0 = t0
			}

			err := c.Push(point.Timestamp, point.Value)
			if err != nil {
				return nil, errors.New(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
			}
		}

		if int64(point.Timestamp) > md.Time {
			md.Time = int64(point.Timestamp)
		}

		// if the last written point was also the last one of the current chunk,
		// or if writeUnfinishedChunks is on, we close the chunk and
		if point.Timestamp == t0+chunkSpan-archiveInfo.SecondsPerPoint || *writeUnfinishedChunks {
			log(fmt.Sprintf("Mark current (last) chunk at t0 %d as finished", t0))
			c.Finish()
			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
		}

		log(fmt.Sprintf("Whisper file %q archive %d (%q) gets %d chunks", file, archiveIdx, name, len(encodedChunks)))
		archives = append(archives, archive.Archive{
			SecondsPerPoint: archiveInfo.SecondsPerPoint,
			Points:          archiveInfo.Points,
			Chunks:          encodedChunks,
			RowKey:          rowKey,
		})
	}

	return &archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
		MetricData:        *md,
		Archives:          archives,
	}, nil
}

func getMetricData(name string, interval int) *schema.MetricData {
	md := &schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: interval,
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	return md
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Andrea Masi. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE.txt file.

package main

import (
	"fmt"
	"os"
	"sort"
)

// Type lessFunc is used for sorting interface date with multiple keys.
type lessFunc func(e1, e2 *interfaceData) bool

// byKey is an helper function that returns a lessFunc type.
// k arguments can be any key declared in makeValueMap.
func byKey(k string) lessFunc {
	return func(e1, e2 *interfaceData) bool {
		return e1.rates[k] < e2.rates[k]
	}
}

// multiSorter implements the sort.Interface.
type multiSorter struct {
	interfaces    []interfaceData
	lessFunctions []lessFunc
}

// sort sorts the slice passed using lessFunc passed to orderBy.
func (ms *multiSorter) sort(d []interfaceData) {
	ms.interfaces = d
	sort.Sort(ms)
}

// orderBy returns a multiSorter loaded with a slice of lessFunc.
func orderBy(lf ...lessFunc) *multiSorter {
	ms := &multiSorter{}
	ms.lessFunctions = lf
	return ms
}

// Len is part of sort.Interface.
func (ms *multiSorter) Len() int {
	return len(ms.interfaces)
}

// Swap is part of sort.Interface.
func (ms *multiSorter) Swap(i, j int) {
	ms.interfaces[i], ms.interfaces[j] = ms.interfaces[j], ms.interfaces[i]
}

// Less is part of sort.Interface.
func (ms *multiSorter) Less(i, j int) bool {
	p, q := &ms.interfaces[i], &ms.interfaces[j]
	var k int
	for k = 0; k < len(ms.lessFunctions)-1; k++ {
		less := ms.lessFunctions[i]
		switch {
		case less(p, q):
			return true
		case less(q, p):
			return false
		}
		// here if p == q, lets try next lessFunc
	}
	// returns result of last lessFunc
	// FIXME this should be always false so can it be avoided?
	//return false
	return ms.lessFunctions[i](p, q)
}

func printHead() {
	fmt.Printf(
		"%20s%12s%9s%9s%12s%12s%12s%12s%12s%12s\n",
		"Host",
		"Interface",
		"Rx-Kb/s",
		"Tx-Kb/s",
		"Rx-Pckts/s",
		"Tx-Pckts/s",
		"Rx-Drp/s",
		"Tx-Drp/s",
		"Rx-Err/s",
		"Tx-Err/s",
	)
}

func displayResults(results []interfaceData, noHead bool) {
	for i, r := range results {
		if i%40 == 0 && !noHead {
			printHead()
		}
		if r.err != nil {
			fmt.Fprintln(os.Stderr, "[ERROR]", r.host, r.err)
		} else {
			fmt.Printf("%20s", r.host)
			fmt.Printf("%12s", r.name)
			fmt.Printf("%9d", uint64(r.rates["rx-Bps"]*8/1024))
			fmt.Printf("%9d", uint64(r.rates["tx-Bps"]*8/1024))
			fmt.Printf("%12d", r.rates["rx-pps"])
			fmt.Printf("%12d", r.rates["tx-pps"])
			fmt.Printf("%12d", r.rates["rx-dps"])
			fmt.Printf("%12d", r.rates["tx-dps"])
			fmt.Printf("%12d", r.rates["rx-eps"])
			fmt.Printf("%12d", r.rates["tx-eps"])
			fmt.Println("")
		}
	}
}
<commit_msg>Fix out of range panic<commit_after>// Copyright (c) 2014 Andrea Masi. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE.txt file.

package main

import (
	"fmt"
	"os"
	"sort"
)

// Type lessFunc is used for sorting interface date with multiple keys.
type lessFunc func(e1, e2 *interfaceData) bool

// byKey is an helper function that returns a lessFunc type.
// k arguments can be any key declared in makeValueMap.
func byKey(k string) lessFunc {
	return func(e1, e2 *interfaceData) bool {
		return e1.rates[k] < e2.rates[k]
	}
}

// multiSorter implements the sort.Interface.
type multiSorter struct {
	interfaces    []interfaceData
	lessFunctions []lessFunc
}

// sort sorts the slice passed using lessFunc passed to orderBy.
func (ms *multiSorter) sort(d []interfaceData) {
	ms.interfaces = d
	sort.Sort(ms)
}

// orderBy returns a multiSorter loaded with a slice of lessFunc.
func orderBy(lf ...lessFunc) *multiSorter {
	ms := &multiSorter{}
	ms.lessFunctions = lf
	return ms
}

// Len is part of sort.Interface.
func (ms *multiSorter) Len() int {
	return len(ms.interfaces)
}

// Swap is part of sort.Interface.
func (ms *multiSorter) Swap(i, j int) {
	ms.interfaces[i], ms.interfaces[j] = ms.interfaces[j], ms.interfaces[i]
}

// Less is part of sort.Interface.
func (ms *multiSorter) Less(i, j int) bool {
	p, q := &ms.interfaces[i], &ms.interfaces[j]
	var k int
	for k = 0; k < len(ms.lessFunctions)-1; k++ {
		less := ms.lessFunctions[k]
		switch {
		case less(p, q):
			return true
		case less(q, p):
			return false
		}
		// here if p == q, lets try next lessFunc
	}
	// returns result of last lessFunc
	// FIXME this should be always false so can it be avoided?
	//return false
	return ms.lessFunctions[k](p, q)
}

func printHead() {
	fmt.Printf(
		"%20s%12s%9s%9s%12s%12s%12s%12s%12s%12s\n",
		"Host",
		"Interface",
		"Rx-Kb/s",
		"Tx-Kb/s",
		"Rx-Pckts/s",
		"Tx-Pckts/s",
		"Rx-Drp/s",
		"Tx-Drp/s",
		"Rx-Err/s",
		"Tx-Err/s",
	)
}

func displayResults(results []interfaceData, noHead bool) {
	for i, r := range results {
		if i%40 == 0 && !noHead {
			printHead()
		}
		if r.err != nil {
			fmt.Fprintln(os.Stderr, "[ERROR]", r.host, r.err)
		} else {
			fmt.Printf("%20s", r.host)
			fmt.Printf("%12s", r.name)
			fmt.Printf("%9d", uint64(r.rates["rx-Bps"]*8/1024))
			fmt.Printf("%9d", uint64(r.rates["tx-Bps"]*8/1024))
			fmt.Printf("%12d", r.rates["rx-pps"])
			fmt.Printf("%12d", r.rates["tx-pps"])
			fmt.Printf("%12d", r.rates["rx-dps"])
			fmt.Printf("%12d", r.rates["tx-dps"])
			fmt.Printf("%12d", r.rates["rx-eps"])
			fmt.Printf("%12d", r.rates["tx-eps"])
			fmt.Println("")
		}
	}
}
<|endoftext|>


<commit_before>package gitnotify

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	githubApp "github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type localGithub struct {
	client GitClient
}

func (*localGithub) WebsiteLink() string {
	return config.GithubURLEndPoint
}

func (*localGithub) RepoLink(repo string) string {
	return fmt.Sprintf(githubRepoEndPoint, repo)
}

func (*localGithub) TreeLink(repo, ref string) string {
	return fmt.Sprintf(githubTreeURLEndPoint, repo, ref)
}

func (*localGithub) CommitLink(repo, ref string) string {
	return fmt.Sprintf(githubCommitURLEndPoint, repo, ref)
}

func (*localGithub) CompareLink(repo, oldCommit, newCommit string) string {
	return fmt.Sprintf(githubCompareURLEndPoint, repo, oldCommit, newCommit)
}

func (g *localGithub) Client() *githubApp.Client {
	return g.client.(*githubApp.Client)
}

// Helper method to create github client
func newGithubClient(token string) *localGithub {
	if token == "" {
		return &localGithub{}
	}
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := githubApp.NewClient(tc)
	var err error
	client.BaseURL, err = url.Parse(config.GithubAPIEndPoint)
	if err != nil {
		log.Println(err)
		return &localGithub{}
	}
	return &localGithub{client}
}

func (g *localGithub) BranchesWithoutRefs(repoName string) ([]string, error) {
	listBranches, err := g.Branches(repoName)
	if err != nil {
		return nil, err
	}

	branches := make([]string, 0, len(listBranches))
	for _, b := range listBranches {
		branches = append(branches, b.Name)
	}
	return branches, nil
}

// caches branch response
func (g *localGithub) Branches(repoName string) ([]*GitRefWithCommit, error) {
	return g.branchTagInfo(repoName, gitRefBranch)
}

// caches branch response
func (g *localGithub) Tags(repoName string) ([]*GitRefWithCommit, error) {
	return g.branchTagInfo(repoName, gitRefTag)
}

func (g *localGithub) DefaultBranch(repoName string) (string, error) {
	ownerRepo := strings.SplitN(repoName, "/", 2)
	repository, gr, err := g.Client().Repositories.Get(ownerRepo[0], ownerRepo[1])

	if err != nil || gr.StatusCode >= 400 {
		// gr will be in case of connection interruption
		// 401 statusCode means the token is no longer valid
		return "", err
	}
	return *repository.DefaultBranch, err
}

func (g *localGithub) branchTagInfo(repoName, option string) ([]*GitRefWithCommit, error) {

	var list []*githubApp.Branch
	var gr *githubApp.Response
	var err error

	refs := make([]*GitRefWithCommit, 0, 100)
	ownerRepo := strings.SplitN(repoName, "/", 2)
	page := 1
	for page != 0 {
		opt := &githubApp.ListOptions{Page: page, PerPage: 100}
		if option == "tags" {
			list, gr, err = g.Client().Repositories.ListBranches(ownerRepo[0], ownerRepo[1], opt)
		} else if option == "branches" {
			list, gr, err = g.Client().Repositories.ListBranches(ownerRepo[0], ownerRepo[1], opt)
		}

		if len(list) == 0 || err != nil || gr.StatusCode >= 400 {
			page = page + 1
			if page >= 100 {
				break
			}
			continue
		}

		for _, r := range list {
			ref := &GitRefWithCommit{
				Name:   *r.Name,
				Commit: *r.Commit.SHA,
			}
			refs = append(refs, ref)
		}
		page = gr.NextPage
	}

	return refs, nil
}

type ghSearchRepo struct {
	Items []*searchRepoItem `json:"items"`
}

// NOTE: the official repo does not support complex queries
// It only supports simple queries. Github API expects data to be in "abc+code:go"
func (g *localGithub) SearchRepos(query string) ([]*searchRepoItem, error) {
	query = g.cleanRepoName(query)
	searchRepositoryURL := fmt.Sprintf("%ssearch/repositories?page=%d&q=%s", config.GithubAPIEndPoint, 1, query)
	req, _ := http.NewRequest("GET", searchRepositoryURL, nil)
	result := new(githubApp.RepositoriesSearchResult)
	gr, err := g.Client().Do(req, result)
	if gr.StatusCode >= 400 {
		return nil, errors.New("status code > 400")
	}

	if err != nil || gr.StatusCode >= 400 {
		return nil, err
	}

	searchResults := make([]*searchRepoItem, 0, len(result.Repositories))
	for _, r := range result.Repositories {
		item := &searchRepoItem{
			ID:   *r.Name,
			Name: *r.FullName,
		}
		if r.Description != nil {
			item.Description = *r.Description
		}
		if r.Homepage != nil {
			item.HomePage = *r.Homepage
		}

		searchResults = append(searchResults, item)
	}
	return searchResults, nil
}

func (g *localGithub) cleanRepoName(search string) string {
	search = strings.Trim(search, " ")
	// Add support for regular searches
	if strings.Contains(search, "/") {
		var modifiedRepoValidator = regexp.MustCompile("[\\p{L}\\d_-]+/[\\.\\p{L}\\d_-]*")
		data := modifiedRepoValidator.FindAllString(search, -1)
		d := strings.Split(data[0], "/")
		rep := fmt.Sprintf("%s+user:%s", d[1], d[0])
		search = strings.Replace(search, data[0], rep, 1)
		search = strings.Trim(search, "/")
	}
	search = strings.Trim(search, " ")
	search = strings.Replace(search, " ", "+", -1)
	return search
}

func (g *localGithub) SearchUsers(query string) ([]*searchUserItem, error) {
	result, gr, err := g.Client().Search.Users(query, nil)

	if err != nil || gr.StatusCode >= 400 {
		return []*searchUserItem{}, err
	}

	searchResults := make([]*searchUserItem, 0, len(result.Users))
	for _, r := range result.Users {
		searchResults = append(searchResults, &searchUserItem{
			ID:    fmt.Sprintf("%d", *r.ID),
			Login: *r.Login,
			Type:  *r.Type,
		})
	}

	return searchResults, nil
}

func (g *localGithub) RemoteOrgType(name string) (string, error) {
	user, gr, err := g.Client().Users.Get(name)

	if err != nil || gr.StatusCode >= 400 {
		return "", err
	}
	// Organization / User
	return *user.Type, nil
}

func (g *localGithub) ReposForUser(organisation string) ([]*searchRepoItem, error) {
	var repoList []*searchRepoItem
	page := 1
	for page != 0 {
		opt := &githubApp.RepositoryListOptions{Sort: "created", ListOptions: githubApp.ListOptions{Page: page, PerPage: 100}}
		repositories, gr, err := g.Client().Repositories.List(organisation, opt)
		if err != nil || gr.StatusCode >= 400 {
			page = page + 1
			if page >= 100 {
				break
			}
			continue
		}
		var repos = make([]*searchRepoItem, 0, len(repositories))
		for _, repo := range repositories {
			item := &searchRepoItem{}
			item.ID = *repo.Name
			item.Name = *repo.Name
			if repo.Homepage != nil {
				item.HomePage = *repo.Homepage
			}
			if repo.Description != nil {
				item.Description = *repo.Description
			}
			repos = append(repos, item)
		}
		repoList = append(repoList, repos...)
		page = gr.NextPage
	}

	return repoList, nil
}
<commit_msg>change Branches and Tags (uses the right api + struct)<commit_after>package gitnotify

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	githubApp "github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

type localGithub struct {
	client GitClient
}

func (*localGithub) WebsiteLink() string {
	return config.GithubURLEndPoint
}

func (*localGithub) RepoLink(repo string) string {
	return fmt.Sprintf(githubRepoEndPoint, repo)
}

func (*localGithub) TreeLink(repo, ref string) string {
	return fmt.Sprintf(githubTreeURLEndPoint, repo, ref)
}

func (*localGithub) CommitLink(repo, ref string) string {
	return fmt.Sprintf(githubCommitURLEndPoint, repo, ref)
}

func (*localGithub) CompareLink(repo, oldCommit, newCommit string) string {
	return fmt.Sprintf(githubCompareURLEndPoint, repo, oldCommit, newCommit)
}

func (g *localGithub) Client() *githubApp.Client {
	return g.client.(*githubApp.Client)
}

// Helper method to create github client
func newGithubClient(token string) *localGithub {
	if token == "" {
		return &localGithub{}
	}
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := githubApp.NewClient(tc)
	var err error
	client.BaseURL, err = url.Parse(config.GithubAPIEndPoint)
	if err != nil {
		log.Println(err)
		return &localGithub{}
	}
	return &localGithub{client}
}

func (g *localGithub) BranchesWithoutRefs(repoName string) ([]string, error) {
	listBranches, err := g.Branches(repoName)
	if err != nil {
		return nil, err
	}

	branches := make([]string, 0, len(listBranches))
	for _, b := range listBranches {
		branches = append(branches, b.Name)
	}
	return branches, nil
}

// caches branch response
func (g *localGithub) Branches(repoName string) ([]*GitRefWithCommit, error) {

	var list []*githubApp.Branch
	var gr *githubApp.Response
	var err error

	refs := make([]*GitRefWithCommit, 0, 100)
	ownerRepo := strings.SplitN(repoName, "/", 2)
	page := 1
	for page != 0 {
		opt := &githubApp.ListOptions{Page: page, PerPage: 100}
		list, gr, err = g.Client().Repositories.ListBranches(ownerRepo[0], ownerRepo[1], opt)

		if len(list) == 0 || err != nil || gr.StatusCode >= 400 {
			page = page + 1
			if page >= 100 {
				break
			}
			continue
		}

		for _, r := range list {
			ref := &GitRefWithCommit{
				Name:   *r.Name,
				Commit: *r.Commit.SHA,
			}
			refs = append(refs, ref)
		}
		page = gr.NextPage
	}

	return refs, nil
}

// caches branch response
// TODO - this is exact duplicate above of except for the list type format
func (g *localGithub) Tags(repoName string) ([]*GitRefWithCommit, error) {
	var list []*githubApp.RepositoryTag
	var gr *githubApp.Response
	var err error

	refs := make([]*GitRefWithCommit, 0, 100)
	ownerRepo := strings.SplitN(repoName, "/", 2)
	page := 1
	for page != 0 {
		opt := &githubApp.ListOptions{Page: page, PerPage: 100}

		list, gr, err = g.Client().Repositories.ListTags(ownerRepo[0], ownerRepo[1], opt)

		if len(list) == 0 || err != nil || gr.StatusCode >= 400 {
			page = page + 1
			if page >= 100 {
				break
			}
			continue
		}

		for _, r := range list {
			ref := &GitRefWithCommit{
				Name:   *r.Name,
				Commit: *r.Commit.SHA,
			}
			refs = append(refs, ref)
		}
		page = gr.NextPage
	}

	return refs, nil
}

func (g *localGithub) DefaultBranch(repoName string) (string, error) {
	ownerRepo := strings.SplitN(repoName, "/", 2)
	repository, gr, err := g.Client().Repositories.Get(ownerRepo[0], ownerRepo[1])

	if err != nil || gr.StatusCode >= 400 {
		// gr will be in case of connection interruption
		// 401 statusCode means the token is no longer valid
		return "", err
	}
	return *repository.DefaultBranch, err
}

type ghSearchRepo struct {
	Items []*searchRepoItem `json:"items"`
}

// NOTE: the official repo does not support complex queries
// It only supports simple queries. Github API expects data to be in "abc+code:go"
func (g *localGithub) SearchRepos(query string) ([]*searchRepoItem, error) {
	query = g.cleanRepoName(query)
	searchRepositoryURL := fmt.Sprintf("%ssearch/repositories?page=%d&q=%s", config.GithubAPIEndPoint, 1, query)
	req, _ := http.NewRequest("GET", searchRepositoryURL, nil)
	result := new(githubApp.RepositoriesSearchResult)
	gr, err := g.Client().Do(req, result)
	if gr.StatusCode >= 400 {
		return nil, errors.New("status code > 400")
	}

	if err != nil || gr.StatusCode >= 400 {
		return nil, err
	}

	searchResults := make([]*searchRepoItem, 0, len(result.Repositories))
	for _, r := range result.Repositories {
		item := &searchRepoItem{
			ID:   *r.Name,
			Name: *r.FullName,
		}
		if r.Description != nil {
			item.Description = *r.Description
		}
		if r.Homepage != nil {
			item.HomePage = *r.Homepage
		}

		searchResults = append(searchResults, item)
	}
	return searchResults, nil
}

func (g *localGithub) cleanRepoName(search string) string {
	search = strings.Trim(search, " ")
	// Add support for regular searches
	if strings.Contains(search, "/") {
		var modifiedRepoValidator = regexp.MustCompile("[\\p{L}\\d_-]+/[\\.\\p{L}\\d_-]*")
		data := modifiedRepoValidator.FindAllString(search, -1)
		d := strings.Split(data[0], "/")
		rep := fmt.Sprintf("%s+user:%s", d[1], d[0])
		search = strings.Replace(search, data[0], rep, 1)
		search = strings.Trim(search, "/")
	}
	search = strings.Trim(search, " ")
	search = strings.Replace(search, " ", "+", -1)
	return search
}

func (g *localGithub) SearchUsers(query string) ([]*searchUserItem, error) {
	result, gr, err := g.Client().Search.Users(query, nil)

	if err != nil || gr.StatusCode >= 400 {
		return []*searchUserItem{}, err
	}

	searchResults := make([]*searchUserItem, 0, len(result.Users))
	for _, r := range result.Users {
		searchResults = append(searchResults, &searchUserItem{
			ID:    fmt.Sprintf("%d", *r.ID),
			Login: *r.Login,
			Type:  *r.Type,
		})
	}

	return searchResults, nil
}

func (g *localGithub) RemoteOrgType(name string) (string, error) {
	user, gr, err := g.Client().Users.Get(name)

	if err != nil || gr.StatusCode >= 400 {
		return "", err
	}
	// Organization / User
	return *user.Type, nil
}

func (g *localGithub) ReposForUser(organisation string) ([]*searchRepoItem, error) {
	var repoList []*searchRepoItem
	page := 1
	for page != 0 {
		opt := &githubApp.RepositoryListOptions{Sort: "created", ListOptions: githubApp.ListOptions{Page: page, PerPage: 100}}
		repositories, gr, err := g.Client().Repositories.List(organisation, opt)
		if err != nil || gr.StatusCode >= 400 {
			page = page + 1
			if page >= 100 {
				break
			}
			continue
		}
		var repos = make([]*searchRepoItem, 0, len(repositories))
		for _, repo := range repositories {
			item := &searchRepoItem{}
			item.ID = *repo.Name
			item.Name = *repo.Name
			if repo.Homepage != nil {
				item.HomePage = *repo.Homepage
			}
			if repo.Description != nil {
				item.Description = *repo.Description
			}
			repos = append(repos, item)
		}
		repoList = append(repoList, repos...)
		page = gr.NextPage
	}

	return repoList, nil
}
<|endoftext|>


<commit_before>package engine

import (
	"fmt"
	"net/url"

	"github.com/keybase/client/go/libkb"
)

type Prereqs struct {
	Session bool
	Device  bool
}

type Engine interface {
	Run(ctx *Context) error
	Prereqs() Prereqs
	libkb.UIConsumer
	G() *libkb.GlobalContext
}

func runPrereqs(e Engine, ctx *Context) (err error) {
	prq := e.Prereqs()

	if prq.Session {
		var ok bool
		ok, err = IsLoggedIn(e, ctx)
		if !ok {
			urlError, isURLError := err.(*url.Error)
			context := ""
			if isURLError {
				context = fmt.Sprintf("Encountered a network error: %s", urlError.Err)
			}
			err = libkb.LoginRequiredError{Context: context}
		}
		if err != nil {
			return err
		}
	}

	if prq.Device {
		var ok bool
		ok, err = IsProvisioned(e, ctx)
		if err != nil {
			return err
		}
		if !ok {
			err = libkb.DeviceRequiredError{}
			return err
		}
	}

	return

}

func RunEngine(e Engine, ctx *Context) error {
	if err := check(e, ctx); err != nil {
		return err
	}
	if err := runPrereqs(e, ctx); err != nil {
		return err
	}
	return e.Run(ctx)
}

func check(c libkb.UIConsumer, ctx *Context) error {
	if err := checkUI(c, ctx); err != nil {
		return CheckError{fmt.Sprintf("%s: %s", c.Name(), err)}
	}

	for _, sub := range c.SubConsumers() {
		if err := check(sub, ctx); err != nil {
			return CheckError{fmt.Sprintf("%s: %s", sub.Name(), err)}
		}
	}

	return nil
}

func checkUI(c libkb.UIConsumer, ctx *Context) error {
	for _, ui := range c.RequiredUIs() {
		if !ctx.HasUI(ui) {
			return CheckError{fmt.Sprintf("requires ui %q", ui)}
		}
	}
	return nil
}
<commit_msg>Add entry/exit debug to RunEngine<commit_after>package engine

import (
	"fmt"
	"net/url"

	"github.com/keybase/client/go/libkb"
)

type Prereqs struct {
	Session bool
	Device  bool
}

type Engine interface {
	Run(ctx *Context) error
	Prereqs() Prereqs
	libkb.UIConsumer
	G() *libkb.GlobalContext
}

func runPrereqs(e Engine, ctx *Context) (err error) {
	prq := e.Prereqs()

	if prq.Session {
		var ok bool
		ok, err = IsLoggedIn(e, ctx)
		if !ok {
			urlError, isURLError := err.(*url.Error)
			context := ""
			if isURLError {
				context = fmt.Sprintf("Encountered a network error: %s", urlError.Err)
			}
			err = libkb.LoginRequiredError{Context: context}
		}
		if err != nil {
			return err
		}
	}

	if prq.Device {
		var ok bool
		ok, err = IsProvisioned(e, ctx)
		if err != nil {
			return err
		}
		if !ok {
			err = libkb.DeviceRequiredError{}
			return err
		}
	}

	return

}

func RunEngine(e Engine, ctx *Context) (err error) {
	e.G().Log.Debug("+ RunEngine(%s)", e.Name())
	defer func() { e.G().Log.Debug("- RunEngine(%s) -> %s", e.Name(), libkb.ErrToOk(err)) }()

	if err = check(e, ctx); err != nil {
		return err
	}
	if err = runPrereqs(e, ctx); err != nil {
		return err
	}
	err = e.Run(ctx)
	return err
}

func check(c libkb.UIConsumer, ctx *Context) error {
	if err := checkUI(c, ctx); err != nil {
		return CheckError{fmt.Sprintf("%s: %s", c.Name(), err)}
	}

	for _, sub := range c.SubConsumers() {
		if err := check(sub, ctx); err != nil {
			return CheckError{fmt.Sprintf("%s: %s", sub.Name(), err)}
		}
	}

	return nil
}

func checkUI(c libkb.UIConsumer, ctx *Context) error {
	for _, ui := range c.RequiredUIs() {
		if !ctx.HasUI(ui) {
			return CheckError{fmt.Sprintf("requires ui %q", ui)}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package teams

import (
	"context"
	"fmt"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

func HandleRotateRequest(ctx context.Context, g *libkb.GlobalContext, teamID keybase1.TeamID, generation keybase1.PerTeamKeyGeneration) (err error) {

	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, fmt.Sprintf("HandleRotateRequest(%s,%d)", teamID, generation), func() error { return err })()

	team, err := Load(ctx, g, keybase1.LoadTeamArg{
		ID:          teamID,
		Public:      teamID.IsPublic(),
		ForceRepoll: true,
	})
	if err != nil {
		return err
	}

	if team.Generation() > generation {
		g.Log.CDebugf(ctx, "current team generation %d > team.clkr generation %d, not rotating", team.Generation(), generation)
		return nil
	}

	g.Log.CDebugf(ctx, "rotating team %s (%s)", team.Name(), teamID)
	if err := team.Rotate(ctx); err != nil {
		g.Log.CDebugf(ctx, "rotating team %s (%s) error: %s", team.Name(), teamID, err)
		return err
	}

	g.Log.CDebugf(ctx, "sucess rotating team %s (%s)", team.Name(), teamID)
	return nil
}

func reloadLocal(ctx context.Context, g *libkb.GlobalContext, row keybase1.TeamChangeRow, change keybase1.TeamChangeSet) (*Team, error) {
	forceRepoll := true
	if change.Renamed {
		// This force reloads the team as a side effect
		err := g.GetTeamLoader().NotifyTeamRename(ctx, row.Id, row.Name)
		if err != nil {
			return nil, err
		}
		forceRepoll = false
	}

	team, err := Load(ctx, g, keybase1.LoadTeamArg{
		ID:          row.Id,
		Public:      row.Id.IsPublic(),
		ForceRepoll: forceRepoll,
	})
	return team, err
}

func handleChangeSingle(ctx context.Context, g *libkb.GlobalContext, row keybase1.TeamChangeRow, change keybase1.TeamChangeSet) (err error) {
	change.KeyRotated = row.KeyRotated
	change.MembershipChanged = row.MembershipChanged

	defer g.CTrace(ctx, fmt.Sprintf("team.handleChangeSingle(%+v, %+v)", row, change), func() error { return err })()

	team, err := reloadLocal(ctx, g, row, change)
	if err != nil {
		return err
	}
	g.NotifyRouter.HandleTeamChanged(ctx, team.ID, team.Name().String(), team.chain().GetLatestSeqno(), change)
	return nil
}

func HandleChangeNotification(ctx context.Context, g *libkb.GlobalContext, rows []keybase1.TeamChangeRow, changes keybase1.TeamChangeSet) (err error) {
	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, "HandleChangeNotification", func() error { return err })()
	for _, row := range rows {
		if err := handleChangeSingle(ctx, g, row, changes); err != nil {
			return err
		}
	}
	return nil
}

func HandleDeleteNotification(ctx context.Context, g *libkb.GlobalContext, rows []keybase1.TeamChangeRow) (err error) {
	defer g.CTrace(ctx, fmt.Sprintf("team.HandleDeleteNotification(%v)", len(rows)), func() error { return err })()

	for _, row := range rows {
		g.Log.CDebugf(ctx, "team.HandleDeleteNotification: (%+v)", row)
		err := g.GetTeamLoader().Delete(ctx, row.Id)
		if err != nil {
			g.Log.CDebugf(ctx, "team.HandleDeleteNotification: error deleting team cache: %v", err)
		}
		g.NotifyRouter.HandleTeamDeleted(ctx, row.Id)
	}
	return nil
}

func HandleExitNotification(ctx context.Context, g *libkb.GlobalContext, rows []keybase1.TeamExitRow) (err error) {
	defer g.CTrace(ctx, fmt.Sprintf("team.HandleExitNotification(%v)", len(rows)), func() error { return err })()

	for _, row := range rows {
		g.Log.CDebugf(ctx, "team.HandleExitNotification: (%+v)", row)
		err := g.GetTeamLoader().Delete(ctx, row.Id)
		if err != nil {
			g.Log.CDebugf(ctx, "team.HandleExitNotification: error deleting team cache: %v", err)
		}
		g.NotifyRouter.HandleTeamExit(ctx, row.Id)
	}
	return nil
}

func HandleSBSRequest(ctx context.Context, g *libkb.GlobalContext, msg keybase1.TeamSBSMsg) (err error) {
	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, "HandleSBSRequest", func() error { return err })()
	for _, invitee := range msg.Invitees {
		if err := handleSBSSingle(ctx, g, msg.TeamID, invitee); err != nil {
			return err
		}
	}
	return nil
}

func handleSBSSingle(ctx context.Context, g *libkb.GlobalContext, teamID keybase1.TeamID, untrustedInviteeFromGregor keybase1.TeamInvitee) (err error) {
	defer g.CTrace(ctx, fmt.Sprintf("team.handleSBSSingle(teamID: %v, invitee: %+v)", teamID, untrustedInviteeFromGregor), func() error { return err })()
	uv := NewUserVersion(untrustedInviteeFromGregor.Uid, untrustedInviteeFromGregor.EldestSeqno)
	req, err := reqFromRole(uv, untrustedInviteeFromGregor.Role)
	if err != nil {
		return err
	}
	req.CompletedInvites = make(map[keybase1.TeamInviteID]keybase1.UserVersionPercentForm)
	req.CompletedInvites[untrustedInviteeFromGregor.InviteID] = uv.PercentForm()

	team, err := Load(ctx, g, keybase1.LoadTeamArg{
		ID:          teamID,
		Public:      teamID.IsPublic(),
		ForceRepoll: true,
	})
	if err != nil {
		return err
	}

	// verify the invite info:

	// find the invite in the team chain
	invite, found := team.chain().FindActiveInviteByID(untrustedInviteeFromGregor.InviteID)
	if !found {
		return libkb.NotFoundError{}
	}
	category, err := invite.Type.C()
	if err != nil {
		return err
	}
	switch category {
	case keybase1.TeamInviteCategory_SBS:
		//  resolve assertion in link (with uid in invite msg)
		ityp, err := invite.Type.String()
		if err != nil {
			return err
		}
		assertion := fmt.Sprintf("%s@%s+uid:%s", string(invite.Name), ityp, untrustedInviteeFromGregor.Uid)

		arg := keybase1.Identify2Arg{
			UserAssertion:    assertion,
			UseDelegateUI:    false,
			Reason:           keybase1.IdentifyReason{Reason: "process team invite"},
			CanSuppressUI:    true,
			IdentifyBehavior: keybase1.TLFIdentifyBehavior_CHAT_GUI,
		}
		ectx := &engine.Context{
			NetContext: ctx,
		}
		eng := engine.NewResolveThenIdentify2(g, &arg)
		if err := engine.RunEngine(eng, ectx); err != nil {
			return err
		}
	case keybase1.TeamInviteCategory_EMAIL:
		// nothing to verify, need to trust the server
	case keybase1.TeamInviteCategory_KEYBASE:
		if err := assertCanAcceptKeybaseInvite(ctx, g, untrustedInviteeFromGregor, invite); err != nil {
			return err
		}
	default:
		return fmt.Errorf("no verification implemented for invite category %s (%+v)", category, invite)
	}

	g.Log.CDebugf(ctx, "checks passed, proceeding with team.ChangeMembership, req = %+v", req)

	if err = team.ChangeMembership(ctx, req); err != nil {
		return err
	}

	// Send chat welcome message
	g.Log.CDebugf(ctx, "sending welcome message for successful SBS handle")
	SendChatInviteWelcomeMessage(ctx, g, team.Name().String(), category, invite.Inviter.Uid,
		untrustedInviteeFromGregor.Uid)

	return nil
}

func assertCanAcceptKeybaseInvite(ctx context.Context, g *libkb.GlobalContext, untrustedInviteeFromGregor keybase1.TeamInvitee, chainInvite keybase1.TeamInvite) error {
	chainUV, err := chainInvite.KeybaseUserVersion()
	if err != nil {
		return err
	}
	if chainUV.Uid.NotEqual(untrustedInviteeFromGregor.Uid) {
		return fmt.Errorf("chain keybase invite link uid %s does not match uid %s in team.sbs message", chainUV.Uid, untrustedInviteeFromGregor.Uid)
	}

	if chainUV.EldestSeqno.Eq(untrustedInviteeFromGregor.EldestSeqno) {
		return nil
	}

	if chainUV.EldestSeqno == 0 {
		g.Log.CDebugf(ctx, "team.sbs invitee eldest seqno: %d, allowing it to take the invite for eldest seqno 0 (reset account)", untrustedInviteeFromGregor.EldestSeqno)
		return nil
	}

	return fmt.Errorf("chain keybase invite link eldest seqno %d does not match eldest seqno %d in team.sbs message", chainUV.EldestSeqno, untrustedInviteeFromGregor.EldestSeqno)
}

func HandleOpenTeamAccessRequest(ctx context.Context, g *libkb.GlobalContext, msg keybase1.TeamOpenReqMsg) (err error) {
	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, "HandleOpenTeamAccessRequest", func() error { return err })()

	team, err := Load(ctx, g, keybase1.LoadTeamArg{
		ID:          msg.TeamID,
		Public:      msg.TeamID.IsPublic(),
		ForceRepoll: true,
	})
	if err != nil {
		return err
	}

	if !team.IsOpen() {
		g.Log.CDebugf(ctx, "team %q is not an open team", team.Name)
		return nil // Not an error - let the handler dismiss the message.
	}

	var req keybase1.TeamChangeReq
	role := team.chain().inner.OpenTeamJoinAs

	for _, tar := range msg.Tars {
		uv := NewUserVersion(tar.Uid, tar.EldestSeqno)
		currentRole, err := team.MemberRole(ctx, uv)
		if err != nil {
			return err
		}

		if currentRole.IsOrAbove(role) {
			continue
		}

		switch role {
		case keybase1.TeamRole_READER:
			req.Readers = append(req.Readers, uv)
		case keybase1.TeamRole_WRITER:
			req.Writers = append(req.Writers, uv)
		default:
			return fmt.Errorf("Unexpected role to add to open team: %v", role)
		}
	}

	return team.ChangeMembership(ctx, req)
}
<commit_msg>add retry to gregor handlers<commit_after>package teams

import (
	"context"
	"fmt"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

func HandleRotateRequest(ctx context.Context, g *libkb.GlobalContext, teamID keybase1.TeamID, generation keybase1.PerTeamKeyGeneration) (err error) {

	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, fmt.Sprintf("HandleRotateRequest(%s,%d)", teamID, generation), func() error { return err })()

	return RetryOnSigOldSeqnoError(ctx, g, func(ctx context.Context, _ int) error {
		team, err := Load(ctx, g, keybase1.LoadTeamArg{
			ID:          teamID,
			Public:      teamID.IsPublic(),
			ForceRepoll: true,
		})
		if err != nil {
			return err
		}

		if team.Generation() > generation {
			g.Log.CDebugf(ctx, "current team generation %d > team.clkr generation %d, not rotating", team.Generation(), generation)
			return nil
		}

		g.Log.CDebugf(ctx, "rotating team %s (%s)", team.Name(), teamID)
		if err := team.Rotate(ctx); err != nil {
			g.Log.CDebugf(ctx, "rotating team %s (%s) error: %s", team.Name(), teamID, err)
			return err
		}

		g.Log.CDebugf(ctx, "sucess rotating team %s (%s)", team.Name(), teamID)
		return nil
	})
}

func reloadLocal(ctx context.Context, g *libkb.GlobalContext, row keybase1.TeamChangeRow, change keybase1.TeamChangeSet) (*Team, error) {
	forceRepoll := true
	if change.Renamed {
		// This force reloads the team as a side effect
		err := g.GetTeamLoader().NotifyTeamRename(ctx, row.Id, row.Name)
		if err != nil {
			return nil, err
		}
		forceRepoll = false
	}

	team, err := Load(ctx, g, keybase1.LoadTeamArg{
		ID:          row.Id,
		Public:      row.Id.IsPublic(),
		ForceRepoll: forceRepoll,
	})
	return team, err
}

func handleChangeSingle(ctx context.Context, g *libkb.GlobalContext, row keybase1.TeamChangeRow, change keybase1.TeamChangeSet) (err error) {
	change.KeyRotated = row.KeyRotated
	change.MembershipChanged = row.MembershipChanged

	defer g.CTrace(ctx, fmt.Sprintf("team.handleChangeSingle(%+v, %+v)", row, change), func() error { return err })()

	team, err := reloadLocal(ctx, g, row, change)
	if err != nil {
		return err
	}
	g.NotifyRouter.HandleTeamChanged(ctx, team.ID, team.Name().String(), team.chain().GetLatestSeqno(), change)
	return nil
}

func HandleChangeNotification(ctx context.Context, g *libkb.GlobalContext, rows []keybase1.TeamChangeRow, changes keybase1.TeamChangeSet) (err error) {
	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, "HandleChangeNotification", func() error { return err })()
	for _, row := range rows {
		if err := handleChangeSingle(ctx, g, row, changes); err != nil {
			return err
		}
	}
	return nil
}

func HandleDeleteNotification(ctx context.Context, g *libkb.GlobalContext, rows []keybase1.TeamChangeRow) (err error) {
	defer g.CTrace(ctx, fmt.Sprintf("team.HandleDeleteNotification(%v)", len(rows)), func() error { return err })()

	for _, row := range rows {
		g.Log.CDebugf(ctx, "team.HandleDeleteNotification: (%+v)", row)
		err := g.GetTeamLoader().Delete(ctx, row.Id)
		if err != nil {
			g.Log.CDebugf(ctx, "team.HandleDeleteNotification: error deleting team cache: %v", err)
		}
		g.NotifyRouter.HandleTeamDeleted(ctx, row.Id)
	}
	return nil
}

func HandleExitNotification(ctx context.Context, g *libkb.GlobalContext, rows []keybase1.TeamExitRow) (err error) {
	defer g.CTrace(ctx, fmt.Sprintf("team.HandleExitNotification(%v)", len(rows)), func() error { return err })()

	for _, row := range rows {
		g.Log.CDebugf(ctx, "team.HandleExitNotification: (%+v)", row)
		err := g.GetTeamLoader().Delete(ctx, row.Id)
		if err != nil {
			g.Log.CDebugf(ctx, "team.HandleExitNotification: error deleting team cache: %v", err)
		}
		g.NotifyRouter.HandleTeamExit(ctx, row.Id)
	}
	return nil
}

func HandleSBSRequest(ctx context.Context, g *libkb.GlobalContext, msg keybase1.TeamSBSMsg) (err error) {
	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, "HandleSBSRequest", func() error { return err })()
	for _, invitee := range msg.Invitees {
		if err := handleSBSSingle(ctx, g, msg.TeamID, invitee); err != nil {
			return err
		}
	}
	return nil
}

func handleSBSSingle(ctx context.Context, g *libkb.GlobalContext, teamID keybase1.TeamID, untrustedInviteeFromGregor keybase1.TeamInvitee) (err error) {
	defer g.CTrace(ctx, fmt.Sprintf("team.handleSBSSingle(teamID: %v, invitee: %+v)", teamID, untrustedInviteeFromGregor), func() error { return err })()
	uv := NewUserVersion(untrustedInviteeFromGregor.Uid, untrustedInviteeFromGregor.EldestSeqno)
	req, err := reqFromRole(uv, untrustedInviteeFromGregor.Role)
	if err != nil {
		return err
	}
	req.CompletedInvites = make(map[keybase1.TeamInviteID]keybase1.UserVersionPercentForm)
	req.CompletedInvites[untrustedInviteeFromGregor.InviteID] = uv.PercentForm()

	return RetryOnSigOldSeqnoError(ctx, g, func(ctx context.Context, _ int) error {
		team, err := Load(ctx, g, keybase1.LoadTeamArg{
			ID:          teamID,
			Public:      teamID.IsPublic(),
			ForceRepoll: true,
		})
		if err != nil {
			return err
		}

		// verify the invite info:

		// find the invite in the team chain
		invite, found := team.chain().FindActiveInviteByID(untrustedInviteeFromGregor.InviteID)
		if !found {
			return libkb.NotFoundError{}
		}
		category, err := invite.Type.C()
		if err != nil {
			return err
		}
		switch category {
		case keybase1.TeamInviteCategory_SBS:
			//  resolve assertion in link (with uid in invite msg)
			ityp, err := invite.Type.String()
			if err != nil {
				return err
			}
			assertion := fmt.Sprintf("%s@%s+uid:%s", string(invite.Name), ityp, untrustedInviteeFromGregor.Uid)

			arg := keybase1.Identify2Arg{
				UserAssertion:    assertion,
				UseDelegateUI:    false,
				Reason:           keybase1.IdentifyReason{Reason: "process team invite"},
				CanSuppressUI:    true,
				IdentifyBehavior: keybase1.TLFIdentifyBehavior_CHAT_GUI,
			}
			ectx := &engine.Context{
				NetContext: ctx,
			}
			eng := engine.NewResolveThenIdentify2(g, &arg)
			if err := engine.RunEngine(eng, ectx); err != nil {
				return err
			}
		case keybase1.TeamInviteCategory_EMAIL:
			// nothing to verify, need to trust the server
		case keybase1.TeamInviteCategory_KEYBASE:
			if err := assertCanAcceptKeybaseInvite(ctx, g, untrustedInviteeFromGregor, invite); err != nil {
				return err
			}
		default:
			return fmt.Errorf("no verification implemented for invite category %s (%+v)", category, invite)
		}

		g.Log.CDebugf(ctx, "checks passed, proceeding with team.ChangeMembership, req = %+v", req)

		if err = team.ChangeMembership(ctx, req); err != nil {
			return err
		}

		// Send chat welcome message
		g.Log.CDebugf(ctx, "sending welcome message for successful SBS handle")
		SendChatInviteWelcomeMessage(ctx, g, team.Name().String(), category, invite.Inviter.Uid,
			untrustedInviteeFromGregor.Uid)

		return nil
	})
}

func assertCanAcceptKeybaseInvite(ctx context.Context, g *libkb.GlobalContext, untrustedInviteeFromGregor keybase1.TeamInvitee, chainInvite keybase1.TeamInvite) error {
	chainUV, err := chainInvite.KeybaseUserVersion()
	if err != nil {
		return err
	}
	if chainUV.Uid.NotEqual(untrustedInviteeFromGregor.Uid) {
		return fmt.Errorf("chain keybase invite link uid %s does not match uid %s in team.sbs message", chainUV.Uid, untrustedInviteeFromGregor.Uid)
	}

	if chainUV.EldestSeqno.Eq(untrustedInviteeFromGregor.EldestSeqno) {
		return nil
	}

	if chainUV.EldestSeqno == 0 {
		g.Log.CDebugf(ctx, "team.sbs invitee eldest seqno: %d, allowing it to take the invite for eldest seqno 0 (reset account)", untrustedInviteeFromGregor.EldestSeqno)
		return nil
	}

	return fmt.Errorf("chain keybase invite link eldest seqno %d does not match eldest seqno %d in team.sbs message", chainUV.EldestSeqno, untrustedInviteeFromGregor.EldestSeqno)
}

func HandleOpenTeamAccessRequest(ctx context.Context, g *libkb.GlobalContext, msg keybase1.TeamOpenReqMsg) (err error) {
	ctx = libkb.WithLogTag(ctx, "CLKR")
	defer g.CTrace(ctx, "HandleOpenTeamAccessRequest", func() error { return err })()

	return RetryOnSigOldSeqnoError(ctx, g, func(ctx context.Context, _ int) error {
		team, err := Load(ctx, g, keybase1.LoadTeamArg{
			ID:          msg.TeamID,
			Public:      msg.TeamID.IsPublic(),
			ForceRepoll: true,
		})
		if err != nil {
			return err
		}

		if !team.IsOpen() {
			g.Log.CDebugf(ctx, "team %q is not an open team", team.Name)
			return nil // Not an error - let the handler dismiss the message.
		}

		var req keybase1.TeamChangeReq
		role := team.chain().inner.OpenTeamJoinAs

		for _, tar := range msg.Tars {
			uv := NewUserVersion(tar.Uid, tar.EldestSeqno)
			currentRole, err := team.MemberRole(ctx, uv)
			if err != nil {
				return err
			}

			if currentRole.IsOrAbove(role) {
				continue
			}

			switch role {
			case keybase1.TeamRole_READER:
				req.Readers = append(req.Readers, uv)
			case keybase1.TeamRole_WRITER:
				req.Writers = append(req.Writers, uv)
			default:
				return fmt.Errorf("Unexpected role to add to open team: %v", role)
			}
		}

		return team.ChangeMembership(ctx, req)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"pool/libary"
	"pool/model"

	"github.com/go-redis/redis/v7"
	"github.com/go-sql-driver/mysql"
)

var (
	jobLen  int = 8
	resLen  int = 5
	poolNum int = 4

	loggers   *log.Logger
	current   int64
	limit     string
	paperKey  string
	listKey   string
	startTime int64
	duration  int64
	statArr   []string
	data      []string
)

func init() {
	logName := fmt.Sprintf("%s-%s.log", "mock_answer_write_pool", time.Now().Format("2006-01-02"))
	logFile, err := os.OpenFile(logName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("打开日志文件失败：", err)
	}

	loggers = log.New(logFile, "", log.Ldate|log.Lmicroseconds)
}

func main() {
	var listSize int = 100
	redisConfig := redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	}

	client := redis.NewClient(&redisConfig)
	_, err := client.Ping().Result()
	if err != nil {
		loggers.Printf("[%s] redis服务不可用,%v\n", "info", err)
		return
	}

	mysqlParams := make(map[string]string)
	mysqlParams["charset"] = "utf8mb4"

	mysqlConfig := mysql.Config{
		User:                 "root",
		Passwd:               "",
		Net:                  "tcp",
		Addr:                 "127.0.0.1:3306",
		DBName:               "test",
		Timeout:              5 * time.Second,
		Collation:            "utf8mb4_unicode_ci",
		Params:               mysqlParams,
		AllowNativePasswords: true,
	}

	mysqlDsn := mysqlConfig.FormatDSN()

	db, err := sql.Open("mysql", mysqlDsn)
	if err != nil {
		loggers.Printf("[%s] dsn:%s open mysql error,%v\n", "error", mysqlDsn, err)
		return
	}

	baseField := []string{"start_time", "duration"}

	startPapersKey := "sx_today_mock_papers"
	opt := &redis.ZRangeBy{Min: "0", Max: "0.0"}

	var workPool *libary.WorkPool = libary.NewWorkPool(poolNum, jobLen, resLen)

	myTimer := time.NewTicker(time.Second)

	processAnswers := func(args ...interface{}) error {
		defer func() {
			if err := recover(); err != nil {
				loggers.Printf("[%s] 出现异常错误,error:%v\n", "error", err)
			}
		}()

		if len := len(args); len < 1 {
			return errors.New("参数错误")
		}

		workIndex := args[len(args)-1].(int)

		answer, ok := args[0].(string)
		if !ok {
			return fmt.Errorf("work_%d answer数据类型转换失败", workIndex)
		}

		loggers.Printf("[%s] work_%d json数据: %s\n", "info", workIndex, answer)

		var mock model.Mock
		err := json.Unmarshal([]byte(answer), &mock)
		if err != nil {
			loggers.Printf("[%s] work_%d %s json数据解析失败,origin:%s %v\n", "error", workIndex, answer, args[0], err)
			return err
		}

		db, ok := args[1].(*sql.DB)
		if !ok {
			return fmt.Errorf("work_%d db数据类型转换失败", workIndex)
		}

		err = mock.ConsumData(db)
		if err != nil {
			loggers.Printf("[%s] work_%d 写入数据库失败,origin:%v %v\n", "error", workIndex, mock, err)
			return err
		}

		return nil
	}

	processStat := func(args ...interface{}) error {
		defer func() {
			if err := recover(); err != nil {
				loggers.Printf("[%s] 出现异常错误,error:%v\n", "error", err)
			}
		}()

		if len := len(args); len < 1 {
			return errors.New("参数错误")
		}

		workIndex := args[len(args)-1].(int)

		examID, ok := args[0].(string)
		if !ok {
			return fmt.Errorf("work_%d examID数据类型转换失败", workIndex)
		}

		paperID, err := strconv.ParseInt(examID, 10, 64)
		if err != nil {
			loggers.Printf("[%s] work_%d %v 转换成int64失败,%v\n", "error", workIndex, examID, err)
			return err
		}

		loggers.Printf("[%s] work_%d paper_id: %v\n", "info", workIndex, paperID)

		db, ok := args[1].(*sql.DB)
		if !ok {
			return fmt.Errorf("work_%d db数据类型转换失败", workIndex)
		}

		mock := &model.Mock{}
		err = mock.UpdateStat(paperID, db)
		if err != nil {
			loggers.Printf("[%s] work_%d %v 更新统计信息失败,%v\n", "error", workIndex, paperID, err)
			fmt.Println(err)
			return err
		}

		return nil
	}

	workPool.Run()

	for {
		now := <-myTimer.C
		current = now.Unix()
		limit = strconv.FormatFloat(float64(current), 'f', 2, 64)
		opt.Max = limit
		papers, err := client.ZRangeByScoreWithScores(startPapersKey, opt).Result()
		if err != nil {
			loggers.Printf("[%s] redis 服务出问题了\n", "info")
			break
		}

		if num := len(papers); num < 1 {
			loggers.Printf("[%s] 没有模考交卷队列数据要处理\n", "info")
			continue
		}

		data = make([]string, 0)
		statArr = make([]string, 0)
		for _, v := range papers {
			paperID := v.Member.(string)
			paperKey = fmt.Sprintf("sx_mock_base_%s", paperID)
			listKey = fmt.Sprintf("sx_mock_answer_list_%s", paperID)
			flag := client.Exists(paperKey).Val()
			listFlag := client.Exists(listKey).Val()
			if flag == 0 {
				client.ZRem(startPapersKey, paperID).Result()
				statArr = append(statArr, paperID)
				continue
			} else {
				paperInfo := client.HMGet(paperKey, baseField...).Val()
				startTime, _ = strconv.ParseInt(paperInfo[0].(string), 10, 64)
				duration, _ = strconv.ParseInt(paperInfo[1].(string), 10, 64)
				if listFlag == 0 && ((startTime + duration) < current) {
					statArr = append(statArr, paperID)
					client.ZRem(startPapersKey, paperID).Result()
					continue
				}
			}

			if listFlag == 1 {
				i := 1
				for {
					if i >= listSize {
						break
					}

					row, err := client.LPop(listKey).Result()
					if err != nil {
						loggers.Printf("[%s] key:%s %v\n", "info", listKey, err)
						break
					}

					data = append(data, row)
					i++
				}
			}

		}

		loggers.Printf("[%s] data len:%d\n", "info", len(data))
		fmt.Println(current, data, len(data))
		if dataNum := len(data); dataNum >= 1 {
			go func(data []string, workPool *libary.WorkPool) {
				for _, m := range data {
					go func(answer string) {
						params := make([]interface{}, 2)
						params[0] = answer
						params[1] = db
						task := &libary.Task{
							F:      processAnswers,
							Params: params,
						}
						go workPool.AddTask(task)
					}(m)
				}
			}(data, workPool)
		}

		if statNum := len(statArr); statNum >= 1 {
			go func(stat_arr []string, workPool *libary.WorkPool) {
				for _, m := range stat_arr {
					go func(workPool *libary.WorkPool, paperID string) {
						params := make([]interface{}, 2)
						params[0] = paperID
						params[1] = db
						task := &libary.Task{
							F:      processStat,
							Params: params,
						}
						go workPool.AddTask(task)
					}(workPool, m)
				}
			}(statArr, workPool)
		}
	}
	/*
		data := make([]string, 19999)
		for i := range data {
			data[i] = fmt.Sprintf("%s_%d", "xiaoming", i)
		}

		var workPool = NewWorkPool(poolNum, jobLen, resLen)
		go func() {
			for _, v := range data {
				go func(workPool *WorksPool, name string) {
					params := make([]interface{}, 1)
					params[0] = name
					task := &Task{
						f: func(args ...interface{}) error {
							name := args[0].(string)
							i := args[1].(int)
							fmt.Printf("worker:%d name:%s\n", i, name)
							return nil
						},
						params: params,
					}
					go workPool.AddTask(task)
				}(workPool, v)
			}
		}()
		workPool.Run()
		for {

		}
	*/
}
<commit_msg>移除注释的代码<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"pool/libary"
	"pool/model"

	"github.com/go-redis/redis/v7"
	"github.com/go-sql-driver/mysql"
)

var (
	jobLen  int = 8
	resLen  int = 5
	poolNum int = 4

	loggers   *log.Logger
	current   int64
	limit     string
	paperKey  string
	listKey   string
	startTime int64
	duration  int64
	statArr   []string
	data      []string
)

func init() {
	logName := fmt.Sprintf("%s-%s.log", "mock_answer_write_pool", time.Now().Format("2006-01-02"))
	logFile, err := os.OpenFile(logName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("打开日志文件失败：", err)
	}

	loggers = log.New(logFile, "", log.Ldate|log.Lmicroseconds)
}

func main() {
	var listSize int = 100
	redisConfig := redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	}

	client := redis.NewClient(&redisConfig)
	_, err := client.Ping().Result()
	if err != nil {
		loggers.Printf("[%s] redis服务不可用,%v\n", "info", err)
		return
	}

	mysqlParams := make(map[string]string)
	mysqlParams["charset"] = "utf8mb4"

	mysqlConfig := mysql.Config{
		User:                 "root",
		Passwd:               "",
		Net:                  "tcp",
		Addr:                 "127.0.0.1:3306",
		DBName:               "test",
		Timeout:              5 * time.Second,
		Collation:            "utf8mb4_unicode_ci",
		Params:               mysqlParams,
		AllowNativePasswords: true,
	}

	mysqlDsn := mysqlConfig.FormatDSN()

	db, err := sql.Open("mysql", mysqlDsn)
	if err != nil {
		loggers.Printf("[%s] dsn:%s open mysql error,%v\n", "error", mysqlDsn, err)
		return
	}

	baseField := []string{"start_time", "duration"}

	startPapersKey := "sx_today_mock_papers"
	opt := &redis.ZRangeBy{Min: "0", Max: "0.0"}

	var workPool *libary.WorkPool = libary.NewWorkPool(poolNum, jobLen, resLen)

	myTimer := time.NewTicker(time.Second)

	processAnswers := func(args ...interface{}) error {
		defer func() {
			if err := recover(); err != nil {
				loggers.Printf("[%s] 出现异常错误,error:%v\n", "error", err)
			}
		}()

		if len := len(args); len < 1 {
			return errors.New("参数错误")
		}

		workIndex := args[len(args)-1].(int)

		answer, ok := args[0].(string)
		if !ok {
			return fmt.Errorf("work_%d answer数据类型转换失败", workIndex)
		}

		loggers.Printf("[%s] work_%d json数据: %s\n", "info", workIndex, answer)

		var mock model.Mock
		err := json.Unmarshal([]byte(answer), &mock)
		if err != nil {
			loggers.Printf("[%s] work_%d %s json数据解析失败,origin:%s %v\n", "error", workIndex, answer, args[0], err)
			return err
		}

		db, ok := args[1].(*sql.DB)
		if !ok {
			return fmt.Errorf("work_%d db数据类型转换失败", workIndex)
		}

		err = mock.ConsumData(db)
		if err != nil {
			loggers.Printf("[%s] work_%d 写入数据库失败,origin:%v %v\n", "error", workIndex, mock, err)
			return err
		}

		return nil
	}

	processStat := func(args ...interface{}) error {
		defer func() {
			if err := recover(); err != nil {
				loggers.Printf("[%s] 出现异常错误,error:%v\n", "error", err)
			}
		}()

		if len := len(args); len < 1 {
			return errors.New("参数错误")
		}

		workIndex := args[len(args)-1].(int)

		examID, ok := args[0].(string)
		if !ok {
			return fmt.Errorf("work_%d examID数据类型转换失败", workIndex)
		}

		paperID, err := strconv.ParseInt(examID, 10, 64)
		if err != nil {
			loggers.Printf("[%s] work_%d %v 转换成int64失败,%v\n", "error", workIndex, examID, err)
			return err
		}

		loggers.Printf("[%s] work_%d paper_id: %v\n", "info", workIndex, paperID)

		db, ok := args[1].(*sql.DB)
		if !ok {
			return fmt.Errorf("work_%d db数据类型转换失败", workIndex)
		}

		mock := &model.Mock{}
		err = mock.UpdateStat(paperID, db)
		if err != nil {
			loggers.Printf("[%s] work_%d %v 更新统计信息失败,%v\n", "error", workIndex, paperID, err)
			fmt.Println(err)
			return err
		}

		return nil
	}

	workPool.Run()

	for {
		now := <-myTimer.C
		current = now.Unix()
		limit = strconv.FormatFloat(float64(current), 'f', 2, 64)
		opt.Max = limit
		papers, err := client.ZRangeByScoreWithScores(startPapersKey, opt).Result()
		if err != nil {
			loggers.Printf("[%s] redis 服务出问题了\n", "info")
			break
		}

		if num := len(papers); num < 1 {
			loggers.Printf("[%s] 没有模考交卷队列数据要处理\n", "info")
			continue
		}

		data = make([]string, 0)
		statArr = make([]string, 0)
		for _, v := range papers {
			paperID := v.Member.(string)
			paperKey = fmt.Sprintf("sx_mock_base_%s", paperID)
			listKey = fmt.Sprintf("sx_mock_answer_list_%s", paperID)
			flag := client.Exists(paperKey).Val()
			listFlag := client.Exists(listKey).Val()
			if flag == 0 {
				client.ZRem(startPapersKey, paperID).Result()
				statArr = append(statArr, paperID)
				continue
			} else {
				paperInfo := client.HMGet(paperKey, baseField...).Val()
				startTime, _ = strconv.ParseInt(paperInfo[0].(string), 10, 64)
				duration, _ = strconv.ParseInt(paperInfo[1].(string), 10, 64)
				if listFlag == 0 && ((startTime + duration) < current) {
					statArr = append(statArr, paperID)
					client.ZRem(startPapersKey, paperID).Result()
					continue
				}
			}

			if listFlag == 1 {
				i := 1
				for {
					if i >= listSize {
						break
					}

					row, err := client.LPop(listKey).Result()
					if err != nil {
						loggers.Printf("[%s] key:%s %v\n", "info", listKey, err)
						break
					}

					data = append(data, row)
					i++
				}
			}

		}

		loggers.Printf("[%s] data len:%d\n", "info", len(data))
		fmt.Println(current, data, len(data))
		if dataNum := len(data); dataNum >= 1 {
			go func(data []string, workPool *libary.WorkPool) {
				for _, m := range data {
					go func(answer string) {
						params := make([]interface{}, 2)
						params[0] = answer
						params[1] = db
						task := &libary.Task{
							F:      processAnswers,
							Params: params,
						}
						go workPool.AddTask(task)
					}(m)
				}
			}(data, workPool)
		}

		if statNum := len(statArr); statNum >= 1 {
			go func(stat_arr []string, workPool *libary.WorkPool) {
				for _, m := range stat_arr {
					go func(workPool *libary.WorkPool, paperID string) {
						params := make([]interface{}, 2)
						params[0] = paperID
						params[1] = db
						task := &libary.Task{
							F:      processStat,
							Params: params,
						}
						go workPool.AddTask(task)
					}(workPool, m)
				}
			}(statArr, workPool)
		}
	}
}
<|endoftext|>


<commit_before>package logger

import (
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	"strings"
)

type Logger interface {
	Level() levels.LogLevel
	Name() string
	FullName() string
	Enabled() map[levels.LogLevel]bool
	Appender() Appender
	Children() []Logger
	Parent() Logger
	GetLogger(string) Logger
	SetLevel(levels.LogLevel)
	Log(levels.LogLevel, ...interface{})
        SetAppender(appender Appender)
}

type logger struct {
	Logger
	level    levels.LogLevel
	name     string
	enabled  map[levels.LogLevel]bool
	appender Appender
	children []Logger
	parent   Logger
}

type Appender interface {
	Write(level levels.LogLevel, message string, args ...interface{})
	SetLayout(layout layout.Layout)
	Layout() layout.Layout
}

func New(name string) Logger {
	l := Logger(&logger{
		level:    levels.DEBUG,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: appenders.Console(),
		children: make([]Logger, 0),
		parent:   nil,
	})
	l.SetLevel(levels.DEBUG)
	return l
}

func unwrap(args ...interface{}) []interface{} {
	head := args[0]
	switch head.(type) {
	case func() (string, []interface{}):
		msg, args := head.(func() (string, []interface{}))()
		args = unwrap(args...)
		return append([]interface{}{msg}, args...)
	case func() []interface{}:
		args = unwrap(head.(func() []interface{})()...)
	case func(...interface{}) []interface{}:
		args = unwrap(head.(func(...interface{}) []interface{})(args[1:]...)...)
	}
	return args
}

func (l *logger) New(name string) Logger {
	lg := Logger(&logger{
		level:    levels.INHERIT,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: nil,
		children: make([]Logger, 0),
		parent:   l,
	})
	l.children = append(l.children, lg)
	return lg
}

func (l *logger) GetLogger(name string) Logger {
	bits := strings.Split(name, ".")

	if l.name == bits[0] {
		if len(bits) == 1 {
			return l
		}

		child := bits[1]
		n := strings.Join(bits[1:], ".")
		for _, c := range l.children {
			if c.Name() == child {
				return c.GetLogger(n)
			}
		}

		lg := l.New(child)
		return lg.GetLogger(n)
	}
	lg := l.New(bits[0])
	return lg.GetLogger(name)
}

func (l *logger) write(level levels.LogLevel, params ...interface{}) {
	a := l.Appender()
	if a != nil {
		a.Write(level, params[0].(string), params[1:]...)
	}
}

func (l *logger) Appender() Appender {
	if a := l.appender; a != nil {
		return a
	}
	if l.parent != nil {
		if a := l.parent.Appender(); a != nil {
			return a
		}
	}
	return nil
}

func (l *logger) Log(level levels.LogLevel, params ...interface{}) {
	if !l.Enabled()[level] {
		return
	}
	l.write(level, unwrap(params...)...)
}

func (l *logger) Level() levels.LogLevel {
	if l.level == levels.INHERIT {
		return l.parent.Level()
	}
	return l.level
}

func (l *logger) Enabled() map[levels.LogLevel]bool {
	if l.level == levels.INHERIT {
		return l.parent.Enabled()
	}
	return l.enabled
}

func (l *logger) Name() string {
	return l.name
}

func (l *logger) FullName() string {
	n := l.name
	if l.parent != nil {
		p := l.parent.FullName()
		if len(p) > 0 {
			n = l.parent.FullName() + "." + n
		}
	}
	return n
}

func (l *logger) SetLevel(level levels.LogLevel) {
	l.level = level
	for k, _ := range levels.LogLevelsToString {
		if k <= level {
			l.enabled[k] = true
		} else {
			l.enabled[k] = false
		}
	}
}

func (l *logger) SetAppender(appender Appender) {
	l.appender = appender
}

func (l *logger) Debug(params ...interface{})   { l.Log(levels.DEBUG, params...) }
func (l *logger) Info(params ...interface{})    { l.Log(levels.INFO, params...) }
func (l *logger) Warn(params ...interface{})    { l.Log(levels.WARN, params...) }
func (l *logger) Error(params ...interface{})   { l.Log(levels.ERROR, params...) }
func (l *logger) Trace(params ...interface{})   { l.Log(levels.TRACE, params...) }
func (l *logger) Printf(params ...interface{})  { l.Log(levels.INFO, params...) }
func (l *logger) Println(params ...interface{}) { l.Log(levels.INFO, params...) }
func (l *logger) Fatalf(params ...interface{})  { l.Log(levels.FATAL, params...) }
<commit_msg>Add missing logger interface funcs<commit_after>package logger

import (
	"strings"

	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
)

type Logger interface {
	Level() levels.LogLevel
	Name() string
	FullName() string
	Enabled() map[levels.LogLevel]bool
	Appender() Appender
	Children() []Logger
	Parent() Logger
	GetLogger(string) Logger
	SetLevel(levels.LogLevel)
	Log(levels.LogLevel, ...interface{})
	SetAppender(appender Appender)

	Debug(params ...interface{})
	Info(params ...interface{})
	Warn(params ...interface{})
	Error(params ...interface{})
	Trace(params ...interface{})
	Printf(params ...interface{})
	Println(params ...interface{})
	Fatalf(params ...interface{})
}

type logger struct {
	Logger
	level    levels.LogLevel
	name     string
	enabled  map[levels.LogLevel]bool
	appender Appender
	children []Logger
	parent   Logger
}

type Appender interface {
	Write(level levels.LogLevel, message string, args ...interface{})
	SetLayout(layout layout.Layout)
	Layout() layout.Layout
}

func New(name string) Logger {
	l := Logger(&logger{
		level:    levels.DEBUG,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: appenders.Console(),
		children: make([]Logger, 0),
		parent:   nil,
	})
	l.SetLevel(levels.DEBUG)
	return l
}

func unwrap(args ...interface{}) []interface{} {
	head := args[0]
	switch head.(type) {
	case func() (string, []interface{}):
		msg, args := head.(func() (string, []interface{}))()
		args = unwrap(args...)
		return append([]interface{}{msg}, args...)
	case func() []interface{}:
		args = unwrap(head.(func() []interface{})()...)
	case func(...interface{}) []interface{}:
		args = unwrap(head.(func(...interface{}) []interface{})(args[1:]...)...)
	}
	return args
}

func (l *logger) New(name string) Logger {
	lg := Logger(&logger{
		level:    levels.INHERIT,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: nil,
		children: make([]Logger, 0),
		parent:   l,
	})
	l.children = append(l.children, lg)
	return lg
}

func (l *logger) GetLogger(name string) Logger {
	bits := strings.Split(name, ".")

	if l.name == bits[0] {
		if len(bits) == 1 {
			return l
		}

		child := bits[1]
		n := strings.Join(bits[1:], ".")
		for _, c := range l.children {
			if c.Name() == child {
				return c.GetLogger(n)
			}
		}

		lg := l.New(child)
		return lg.GetLogger(n)
	}
	lg := l.New(bits[0])
	return lg.GetLogger(name)
}

func (l *logger) write(level levels.LogLevel, params ...interface{}) {
	a := l.Appender()
	if a != nil {
		a.Write(level, params[0].(string), params[1:]...)
	}
}

func (l *logger) Appender() Appender {
	if a := l.appender; a != nil {
		return a
	}
	if l.parent != nil {
		if a := l.parent.Appender(); a != nil {
			return a
		}
	}
	return nil
}

func (l *logger) Log(level levels.LogLevel, params ...interface{}) {
	if !l.Enabled()[level] {
		return
	}
	l.write(level, unwrap(params...)...)
}

func (l *logger) Level() levels.LogLevel {
	if l.level == levels.INHERIT {
		return l.parent.Level()
	}
	return l.level
}

func (l *logger) Enabled() map[levels.LogLevel]bool {
	if l.level == levels.INHERIT {
		return l.parent.Enabled()
	}
	return l.enabled
}

func (l *logger) Name() string {
	return l.name
}

func (l *logger) FullName() string {
	n := l.name
	if l.parent != nil {
		p := l.parent.FullName()
		if len(p) > 0 {
			n = l.parent.FullName() + "." + n
		}
	}
	return n
}

func (l *logger) SetLevel(level levels.LogLevel) {
	l.level = level
	for k, _ := range levels.LogLevelsToString {
		if k <= level {
			l.enabled[k] = true
		} else {
			l.enabled[k] = false
		}
	}
}

func (l *logger) SetAppender(appender Appender) {
	l.appender = appender
}

func (l *logger) Debug(params ...interface{})   { l.Log(levels.DEBUG, params...) }
func (l *logger) Info(params ...interface{})    { l.Log(levels.INFO, params...) }
func (l *logger) Warn(params ...interface{})    { l.Log(levels.WARN, params...) }
func (l *logger) Error(params ...interface{})   { l.Log(levels.ERROR, params...) }
func (l *logger) Trace(params ...interface{})   { l.Log(levels.TRACE, params...) }
func (l *logger) Printf(params ...interface{})  { l.Log(levels.INFO, params...) }
func (l *logger) Println(params ...interface{}) { l.Log(levels.INFO, params...) }
func (l *logger) Fatalf(params ...interface{})  { l.Log(levels.FATAL, params...) }
<|endoftext|>


<commit_before>package opentracing

import (
	"context"
	"github.com/opentracing/opentracing-go"
)

// EndpointOptions holds the options for tracing an endpoint
type EndpointOptions struct {
	// IgnoreBusinessError if set to true will not treat a business error
	// identified through the endpoint.Failer interface as a span error.
	IgnoreBusinessError bool

	// GetOperationName is an optional function that can set the span operation name based on the existing one
	// for the endpoint and information in the context.
	//
	// If the function is nil, or the returned name is empty, the existing name for the endpoint is used.
	GetOperationName func(ctx context.Context, name string) string

	// Tags holds the default tags which will be set on span
	// creation by our Endpoint middleware.
	Tags opentracing.Tags

	// GetTags is an optional function that can extract trace tags
	// from the context and add them to the span.
	GetTags func(ctx context.Context) opentracing.Tags
}

// EndpointOption allows for functional options to Opentracing endpoint
// tracing middleware.
type EndpointOption func(*EndpointOptions)

// WithOptions sets all configuration options at once by use of the
// EndpointOptions struct.
func WithOptions(options EndpointOptions) EndpointOption {
	return func(o *EndpointOptions) {
		*o = options
	}
}

// WithIgnoreBusinessError if set to true will not treat a business error
// identified through the endpoint.Failer interface as a span error.
func WithIgnoreBusinessError(ignoreBusinessError bool) EndpointOption {
	return func(o *EndpointOptions) {
		o.IgnoreBusinessError = ignoreBusinessError
	}
}

// WithOperationName allows to set function that can set the span operation name based on the existing one
// for the endpoint and information in the context.
func WithOperationName(getOperationName func(ctx context.Context, name string) string) EndpointOption {
	return func(o *EndpointOptions) {
		o.GetOperationName = getOperationName
	}
}

// WithTags sets the default tags for the spans created by the Endpoint tracer.
func WithTags(tags opentracing.Tags) EndpointOption {
	return func(o *EndpointOptions) {
		o.Tags = tags
	}
}

// WithExtraTags extracts additional attributes from the context.
func WithExtraTags(getTags func(ctx context.Context) opentracing.Tags) EndpointOption {
	return func(o *EndpointOptions) {
		o.GetTags = getTags
	}
}
<commit_msg>update comments<commit_after>package opentracing

import (
	"context"
	"github.com/opentracing/opentracing-go"
)

// EndpointOptions holds the options for tracing an endpoint
type EndpointOptions struct {
	// IgnoreBusinessError if set to true will not treat a business error
	// identified through the endpoint.Failer interface as a span error.
	IgnoreBusinessError bool

	// GetOperationName is an optional function that can set the span operation name based on the existing one
	// for the endpoint and information in the context.
	//
	// If the function is nil, or the returned name is empty, the existing name for the endpoint is used.
	GetOperationName func(ctx context.Context, name string) string

	// Tags holds the default tags which will be set on span
	// creation by our Endpoint middleware.
	Tags opentracing.Tags

	// GetTags is an optional function that can extract tags
	// from the context and add them to the span.
	GetTags func(ctx context.Context) opentracing.Tags
}

// EndpointOption allows for functional options to endpoint tracing middleware.
type EndpointOption func(*EndpointOptions)

// WithOptions sets all configuration options at once by use of the EndpointOptions struct.
func WithOptions(options EndpointOptions) EndpointOption {
	return func(o *EndpointOptions) {
		*o = options
	}
}

// WithIgnoreBusinessError if set to true will not treat a business error
// identified through the endpoint.Failer interface as a span error.
func WithIgnoreBusinessError(ignoreBusinessError bool) EndpointOption {
	return func(o *EndpointOptions) {
		o.IgnoreBusinessError = ignoreBusinessError
	}
}

// WithOperationName allows to set function that can set the span operation name based on the existing one
// for the endpoint and information in the context.
func WithOperationName(getOperationName func(ctx context.Context, name string) string) EndpointOption {
	return func(o *EndpointOptions) {
		o.GetOperationName = getOperationName
	}
}

// WithTags sets the default tags for the spans created by the Endpoint tracer.
func WithTags(tags opentracing.Tags) EndpointOption {
	return func(o *EndpointOptions) {
		o.Tags = tags
	}
}

// WithExtraTags extracts additional tags from the context.
func WithExtraTags(getTags func(ctx context.Context) opentracing.Tags) EndpointOption {
	return func(o *EndpointOptions) {
		o.GetTags = getTags
	}
}
<|endoftext|>