<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	defaultPagination := &piazza.JsonPagination{
		PerPage: 10,
		Page:    0,
		Order:   piazza.PaginationOrderDescending,
		SortBy:  "createdOn",
	}
	pagination, err := piazza.NewJsonPagination(params, defaultPagination)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	log.Printf("dsl: %s", dsl)

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, pagination)
	} else {
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, dsl)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	// TODO: unsafe truncation
	matched := int(searchResult.NumberMatched())
	var lines []Message

	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}

		var msg Message
		err = json.Unmarshal(*hit.Source, msg)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			return service.newInternalErrorResponse(err)
		}

		// just in case
		err = msg.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines = append(lines, msg)
	}

	pagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.AsString("service", nil)
	if err != nil {
		return "", err
	}

	contains, err := params.AsString("contains", nil)
	if err != nil {
		return "", err
	}

	before, err := params.AsTime("before", nil)
	if err != nil {
		return "", err
	}

	after, err := params.AsTime("after", nil)
	if err != nil {
		return "", err
	}

	if service != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": *service,
			},
		})
	}

	if contains != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  contains,
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if after != nil || before != nil {
		rangeParams := map[string]time.Time{}

		if after != nil {
			rangeParams["gte"] = *after
		}

		if before != nil {
			rangeParams["lte"] = *before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"createdOn": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}
<commit_msg>Fix pointer typo<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	defaultPagination := &piazza.JsonPagination{
		PerPage: 10,
		Page:    0,
		Order:   piazza.PaginationOrderDescending,
		SortBy:  "createdOn",
	}
	pagination, err := piazza.NewJsonPagination(params, defaultPagination)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	log.Printf("dsl: %s", dsl)

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, pagination)
	} else {
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, dsl)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	// TODO: unsafe truncation
	matched := int(searchResult.NumberMatched())
	var lines []Message

	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}

		var msg *Message
		err = json.Unmarshal(*hit.Source, msg)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			return service.newInternalErrorResponse(err)
		}

		// just in case
		err = msg.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines = append(lines, msg)
	}

	pagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.AsString("service", nil)
	if err != nil {
		return "", err
	}

	contains, err := params.AsString("contains", nil)
	if err != nil {
		return "", err
	}

	before, err := params.AsTime("before", nil)
	if err != nil {
		return "", err
	}

	after, err := params.AsTime("after", nil)
	if err != nil {
		return "", err
	}

	if service != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": *service,
			},
		})
	}

	if contains != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  contains,
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if after != nil || before != nil {
		rangeParams := map[string]time.Time{}

		if after != nil {
			rangeParams["gte"] = *after
		}

		if before != nil {
			rangeParams["lte"] = *before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"createdOn": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}
<|endoftext|>


<commit_before>package logs_test

import (
	"errors"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/cloudfoundry/noaa/events"
	"github.com/pivotal-cf-experimental/diego-edge-cli/logs"
)

type fakeConsumer struct {
	pendingLogs   []*events.LogMessage
	pendingErrors []error
}

func (consumer *fakeConsumer) TailingLogs(appGuid string, authToken string, outputChan chan<- *events.LogMessage, errorChan chan<- error, stopChan chan struct{}) {
	for _, logMessage := range consumer.pendingLogs {
		outputChan <- logMessage
	}

	for _, err := range consumer.pendingErrors {
		errorChan <- err
	}
	close(errorChan)
}

func (consumer *fakeConsumer) addToPendingLogs(logMessage *events.LogMessage) {
	consumer.pendingLogs = append(consumer.pendingLogs, logMessage)
}

func (consumer *fakeConsumer) addToPendingErrors(err error) {
	consumer.pendingErrors = append(consumer.pendingErrors, err)
}

var _ = Describe("logs", func() {
	Describe("tailing logs", func() {
		It("uses the logMessage callback", func() {
			consumer := &fakeConsumer{pendingLogs: []*events.LogMessage{}, pendingErrors: []error{}}
			logReader := logs.NewLogReader(consumer)

			consumer.addToPendingLogs(&events.LogMessage{Message: []byte("Message 1")})
			consumer.addToPendingLogs(&events.LogMessage{Message: []byte("Message 2")})

			recievedLogs := []string{}
			responseFunc := func(log string) {
				recievedLogs = append(recievedLogs, log)
			}

			logReader.TailLogs("app-guid", responseFunc)

			Expect(recievedLogs).To(Equal([]string{"Message 1", "Message 2"}))
		})

		It("uses the logMessage callback", func() {
			consumer := &fakeConsumer{pendingErrors: []error{}}
			logReader := logs.NewLogReader(consumer)

			consumer.addToPendingErrors(errors.New("error 1"))
			consumer.addToPendingErrors(errors.New("error 2"))

			recievedLogs := []string{}
			responseFunc := func(log string) {
				recievedLogs = append(recievedLogs, log)
			}

			logReader.TailLogs("app-guid", responseFunc)

			Expect(recievedLogs).To(Equal([]string{"error 1", "error 2"}))
		})
	})

})
<commit_msg>Rename recievedLogs to receivedLogs<commit_after>package logs_test

import (
	"errors"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/cloudfoundry/noaa/events"
	"github.com/pivotal-cf-experimental/diego-edge-cli/logs"
)

type fakeConsumer struct {
	pendingLogs   []*events.LogMessage
	pendingErrors []error
}

func (consumer *fakeConsumer) TailingLogs(appGuid string, authToken string, outputChan chan<- *events.LogMessage, errorChan chan<- error, stopChan chan struct{}) {
	for _, logMessage := range consumer.pendingLogs {
		outputChan <- logMessage
	}

	for _, err := range consumer.pendingErrors {
		errorChan <- err
	}
	close(errorChan)
}

func (consumer *fakeConsumer) addToPendingLogs(logMessage *events.LogMessage) {
	consumer.pendingLogs = append(consumer.pendingLogs, logMessage)
}

func (consumer *fakeConsumer) addToPendingErrors(err error) {
	consumer.pendingErrors = append(consumer.pendingErrors, err)
}

var _ = Describe("logs", func() {
	Describe("tailing logs", func() {
		It("uses the logMessage callback", func() {
			consumer := &fakeConsumer{pendingLogs: []*events.LogMessage{}, pendingErrors: []error{}}
			logReader := logs.NewLogReader(consumer)

			consumer.addToPendingLogs(&events.LogMessage{Message: []byte("Message 1")})
			consumer.addToPendingLogs(&events.LogMessage{Message: []byte("Message 2")})

			receivedLogs := []string{}
			responseFunc := func(log string) {
				receivedLogs = append(receivedLogs, log)
			}

			logReader.TailLogs("app-guid", responseFunc)

			Expect(receivedLogs).To(Equal([]string{"Message 1", "Message 2"}))
		})

		It("uses the logMessage callback", func() {
			consumer := &fakeConsumer{pendingErrors: []error{}}
			logReader := logs.NewLogReader(consumer)

			consumer.addToPendingErrors(errors.New("error 1"))
			consumer.addToPendingErrors(errors.New("error 2"))

			receivedLogs := []string{}
			responseFunc := func(log string) {
				receivedLogs = append(receivedLogs, log)
			}

			logReader.TailLogs("app-guid", responseFunc)

			Expect(receivedLogs).To(Equal([]string{"error 1", "error 2"}))
		})
	})

})
<|endoftext|>


<commit_before>package lzma

import (
	"github.com/uli-go/xz/hash"
)

/* For compression we need to find byte sequences that match the current byte
 * byte sequences in the available dictionary.
 *
 * The simplest way to achieve that are hash tables. While the hash table
 * implementation shouldn't know that, it will support hashes for two-byte and
 * four-byte strings. A hash is a uint64 number and the hash table maps that to
 * a uint32 value.
 */

// slotEntries gives the number of entries in one slot of the hash table. If
// slotEntries is larger than 128 the representation of fields i_ and j_ in
// slot must be reworked.
const slotEntries = 24

// The minTableExponent give the minimum and maximum for the table exponent.
// The minimum is somehow arbitrary but the maximum is defined by the number of
// bits in the hash value.
const (
	minTableExponent = 9
	maxTableExponent = 64
)

// slot defines the data structure for a slot in the hash table. The number of
// entries is given by slotEntries constant.
type slot struct {
	entries [slotEntries]uint32
	// start index; bit 7 set if non-empty
	i_ uint8
	// next entry to overwrite
	j_ uint8
}

func (s *slot) start() int {
	return int(s.i_ & 0x7f)
}

func (s *slot) end() int {
	return int(s.j_)
}

func (s *slot) empty() bool {
	return s.i_&0x80 == 0
}

// getEntries returns all entries of a slot in the sequence that they have been
// entered.
func (s *slot) getEntries() []uint32 {
	if s.empty() {
		return nil
	}
	r := make([]uint32, 0, slotEntries)
	i, j := s.start(), s.end()
	if i >= j {
		r = append(r, s.entries[i:]...)
		r = append(r, s.entries[:j]...)
	} else {
		r = append(r, s.entries[i:j]...)
	}
	return r
}

// putEntry puts an entry into a slot.
func (s *slot) putEntry(p uint32) {
	i, j := s.start(), s.end()
	s.entries[j] = p
	jp1 := (j + 1) % slotEntries
	if j == i && !s.empty() {
		j = jp1
		i = j
	} else {
		j = jp1
	}
	s.i_ = 0x80 | uint8(i)
	s.j_ = uint8(j)
}

// hashTable stores the hash table including the rolling hash method.
type hashTable struct {
	t        []slot
	exponent int
	mask     uint64
	h        hash.Roller
}

// newHashTable creates a new hash table.
func newHashTable(exponent int, h hash.Roller) *hashTable {
	if !(minTableExponent <= exponent && exponent <= maxTableExponent) {
		panic("argument exponent out of range")
	}
	if h == nil {
		panic("argument h is nil")
	}
	slotLen := int(1) << uint(exponent)
	if slotLen <= 0 {
		panic("exponent is too large")
	}
	return &hashTable{
		t:        make([]slot, slotLen),
		exponent: exponent,
		mask:     (uint64(1) << uint(exponent)) - 1,
		h:        h}
}

// lookup of a byte sequence. b must have at least the length as required by
// the hash function and should have the correct length.
func (t *hashTable) lookup(b []byte) []uint32 {
	h := t.h.Hashes(b)[0]
	return t.getEntries(h)
}

// putEntry puts an entry into the hash table using the given hash.
func (t *hashTable) putEntry(h uint64, p uint32) {
	t.t[h&t.mask].putEntry(p)
}

// getEntries returns all the values that cant be found in the hash table.
func (t *hashTable) getEntries(h uint64) []uint32 {
	return t.t[h&t.mask].getEntries()
}

// put puts the hash value for a byte sequence into the hash table. b should
// have the length as supported by the rolling hash.
func (t *hashTable) put(b []byte, p uint32) {
	h := t.h.Hashes(b)[0]
	t.putEntry(h, p)
}

// puts all positions for the byte sequence into the hash table. The sequences
// following the first one, will have the p value increased with the offset in
// the byte sequence.
func (t *hashTable) putAll(b []byte, p uint32) {
	hashes := t.h.Hashes(b)
	for i, h := range hashes {
		t.putEntry(h, p+uint32(i))
	}
}
<commit_msg>lzma: added comments for new functions in hashtable.go<commit_after>package lzma

import (
	"github.com/uli-go/xz/hash"
)

/* For compression we need to find byte sequences that match the current byte
 * byte sequences in the available dictionary.
 *
 * The simplest way to achieve that are hash tables. While the hash table
 * implementation shouldn't know that, it will support hashes for two-byte and
 * four-byte strings. A hash is a uint64 number and the hash table maps that to
 * a uint32 value.
 */

// slotEntries gives the number of entries in one slot of the hash table. If
// slotEntries is larger than 128 the representation of fields i_ and j_ in
// slot must be reworked.
const slotEntries = 24

// The minTableExponent give the minimum and maximum for the table exponent.
// The minimum is somehow arbitrary but the maximum is defined by the number of
// bits in the hash value.
const (
	minTableExponent = 9
	maxTableExponent = 64
)

// slot defines the data structure for a slot in the hash table. The number of
// entries is given by slotEntries constant.
type slot struct {
	entries [slotEntries]uint32
	// start index; bit 7 set if non-empty
	i_ uint8
	// next entry to overwrite
	j_ uint8
}

// start retuns the start index of the slot
func (s *slot) start() int {
	return int(s.i_ & 0x7f)
}

// end returns the end index of the slot
func (s *slot) end() int {
	return int(s.j_)
}

// empty returns true if nothing is stored in the slot
func (s *slot) empty() bool {
	return s.i_&0x80 == 0
}

// getEntries returns all entries of a slot in the sequence that they have been
// entered.
func (s *slot) getEntries() []uint32 {
	if s.empty() {
		return nil
	}
	r := make([]uint32, 0, slotEntries)
	i, j := s.start(), s.end()
	if i >= j {
		r = append(r, s.entries[i:]...)
		r = append(r, s.entries[:j]...)
	} else {
		r = append(r, s.entries[i:j]...)
	}
	return r
}

// putEntry puts an entry into a slot.
func (s *slot) putEntry(p uint32) {
	i, j := s.start(), s.end()
	s.entries[j] = p
	jp1 := (j + 1) % slotEntries
	if j == i && !s.empty() {
		j = jp1
		i = j
	} else {
		j = jp1
	}
	s.i_ = 0x80 | uint8(i)
	s.j_ = uint8(j)
}

// hashTable stores the hash table including the rolling hash method.
type hashTable struct {
	t        []slot
	exponent int
	mask     uint64
	h        hash.Roller
}

// newHashTable creates a new hash table.
func newHashTable(exponent int, h hash.Roller) *hashTable {
	if !(minTableExponent <= exponent && exponent <= maxTableExponent) {
		panic("argument exponent out of range")
	}
	if h == nil {
		panic("argument h is nil")
	}
	slotLen := int(1) << uint(exponent)
	if slotLen <= 0 {
		panic("exponent is too large")
	}
	return &hashTable{
		t:        make([]slot, slotLen),
		exponent: exponent,
		mask:     (uint64(1) << uint(exponent)) - 1,
		h:        h}
}

// lookup of a byte sequence. b must have at least the length as required by
// the hash function and should have the correct length.
func (t *hashTable) lookup(b []byte) []uint32 {
	h := t.h.Hashes(b)[0]
	return t.getEntries(h)
}

// putEntry puts an entry into the hash table using the given hash.
func (t *hashTable) putEntry(h uint64, p uint32) {
	t.t[h&t.mask].putEntry(p)
}

// getEntries returns all the values that cant be found in the hash table.
func (t *hashTable) getEntries(h uint64) []uint32 {
	return t.t[h&t.mask].getEntries()
}

// put puts the hash value for a byte sequence into the hash table. b should
// have the length as supported by the rolling hash.
func (t *hashTable) put(b []byte, p uint32) {
	h := t.h.Hashes(b)[0]
	t.putEntry(h, p)
}

// puts all positions for the byte sequence into the hash table. The sequences
// following the first one, will have the p value increased with the offset in
// the byte sequence.
func (t *hashTable) putAll(b []byte, p uint32) {
	hashes := t.h.Hashes(b)
	for i, h := range hashes {
		t.putEntry(h, p+uint32(i))
	}
}
<|endoftext|>


<commit_before>package fs

import (
	"fmt"
	"hash"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/pkg/xattr"
)

// boolTrueHashValue is used when we need to write something indicating a bool in the input.
var boolTrueHashValue = []byte{2}

// A PathHasher is responsible for hashing & remembering paths.
type PathHasher struct {
	new       func() hash.Hash
	memo      map[string][]byte
	wait      map[string]*pendingHash
	mutex     sync.RWMutex
	root      string
	xattrName string
	useXattrs bool
}

type pendingHash struct {
	Ch   chan struct{}
	Hash []byte
	Err  error
}

// NewPathHasher returns a new PathHasher based on the given root directory.
func NewPathHasher(root string, useXattrs bool, hash func() hash.Hash, hashSuffix string) *PathHasher {
	return &PathHasher{
		new:       hash,
		memo:      map[string][]byte{},
		wait:      map[string]*pendingHash{},
		root:      root,
		useXattrs: useXattrs,
		xattrName: "user.plz_hash" + hashSuffix,
	}
}

// Size returns the size of the hash this hasher will return, in bytes.
func (hasher *PathHasher) Size() int {
	return hasher.new().Size()
}

// NewHash returns a new hash.Hash instance from this hasher.
func (hasher *PathHasher) NewHash() hash.Hash {
	return hasher.new()
}

// DisableXattrs turns off xattr support, which bypasses using them to record file hashes.
func (hasher *PathHasher) DisableXattrs() {
	hasher.useXattrs = false
}

// Hash hashes a single path.
// It is memoised and so will only hash each path once, unless recalc is true which will
// then force a recalculation of it.
// If store is true then the hash may be stored permanently; this should not be set for files that
// are user-controlled.
// TODO(peterebden): ensure that xattrs are marked correctly on cache retrieval.
func (hasher *PathHasher) Hash(path string, recalc, store bool) ([]byte, error) {
	path = hasher.ensureRelative(path)
	if !recalc {
		hasher.mutex.RLock()
		cached, present := hasher.memo[path]
		hasher.mutex.RUnlock()
		if present {
			return cached, nil
		}
	}
	// This check is important; if the file doesn't exist now, we don't want that
	// recorded forever in hasher.wait since it might get created later.
	if !PathExists(path) {
		return nil, fmt.Errorf("cannot calculate hash for %s: %s", path, os.ErrNotExist)
	}
	hasher.mutex.Lock()
	if pending, present := hasher.wait[path]; present {
		hasher.mutex.Unlock()
		<-pending.Ch
		return pending.Hash, pending.Err
	}
	pending := &pendingHash{Ch: make(chan struct{})}
	hasher.wait[path] = pending
	hasher.mutex.Unlock()
	result, err := hasher.hash(path, store, !recalc)
	hasher.mutex.Lock()
	if err == nil {
		hasher.memo[path] = result
	}
	delete(hasher.wait, path)
	hasher.mutex.Unlock()
	pending.Hash = result
	pending.Err = err
	close(pending.Ch)
	return result, err
}

// MustHash is as Hash but panics on error.
func (hasher *PathHasher) MustHash(path string) []byte {
	hash, err := hasher.Hash(path, false, false)
	if err != nil {
		panic(err)
	}
	return hash
}

// MoveHash is used when we move files from tmp to out and there was one there before; that's
// the only case in which the hash of a filepath could change.
func (hasher *PathHasher) MoveHash(oldPath, newPath string, copy bool) {
	oldPath = hasher.ensureRelative(oldPath)
	newPath = hasher.ensureRelative(newPath)
	hasher.mutex.Lock()
	defer hasher.mutex.Unlock()
	if oldHash, present := hasher.memo[oldPath]; present {
		hasher.memo[newPath] = oldHash
		// If the path is in plz-out/tmp we aren't ever going to use it again, so free some space.
		if !copy && strings.HasPrefix(oldPath, "plz-out/tmp") {
			delete(hasher.memo, oldPath)
		}
	}
}

// SetHash is used to directly set a hash for a path.
// This is used for remote files where we download them & therefore know the hash as they come in.
// TODO(peterebden): We should probably use this more for things like caches and so forth...
func (hasher *PathHasher) SetHash(path string, hash []byte) {
	hasher.mutex.Lock()
	hasher.memo[path] = hash
	hasher.mutex.Unlock()
	hasher.storeHash(path, hash)
}

func (hasher *PathHasher) hash(path string, store, read bool) ([]byte, error) {
	// Try to read xattrs first so we don't have to hash the whole thing.
	if read && strings.HasPrefix(path, "plz-out/") && hasher.useXattrs {
		if b, err := xattr.LGet(path, hasher.xattrName); err == nil {
			return b, nil
		}
	}
	h := hasher.new()
	info, err := os.Lstat(path)
	if err == nil && info.Mode()&os.ModeSymlink != 0 {
		// Handle symlinks specially (don't attempt to read their contents).
		dest, err := os.Readlink(path)
		if err != nil {
			return nil, err
		}
		// Write something arbitrary indicating this is a symlink.
		// This isn't quite perfect - it could potentially get mixed up with a file with the
		// appropriate contents, but that is not really likely.
		h.Write(boolTrueHashValue)
		if rel := hasher.ensureRelative(dest); (rel != dest || !filepath.IsAbs(dest)) && !filepath.IsAbs(path) {
			// Inside the root of our repo so it's something we manage - just hash its (relative) destination
			h.Write([]byte(rel))
		} else {
			// Outside the repo; it's a system tool, so we hash its contents.
			err := hasher.fileHash(h, path)
			return h.Sum(nil), err
		}
		return h.Sum(nil), nil
	} else if err == nil && info.IsDir() {
		err = WalkMode(path, func(p string, isDir bool, mode os.FileMode) error {
			if mode&os.ModeSymlink != 0 {
				// Is a symlink, must verify that it's not a link outside the tmp dir.
				deref, err := filepath.EvalSymlinks(p)
				if err != nil {
					return err
				}
				if !strings.HasPrefix(deref, path) {
					return fmt.Errorf("Output %s links outside the build dir (to %s)", p, deref)
				}
				// Deliberately do not attempt to read it. We will read the contents later since
				// it is a link within the temp dir anyway, and if it's a link to a directory
				// it can introduce a cycle.
				// Just write something to the hash indicating that we found something here,
				// otherwise rules might be marked as unchanged if they added additional symlinks.
				h.Write(boolTrueHashValue)
			} else if !isDir {
				return hasher.fileHash(h, p)
			}
			return nil
		})
	} else {
		err = hasher.fileHash(h, path) // let this handle any other errors
	}
	hash := h.Sum(nil)
	if err != nil {
		return hash, err
	} else if store && hasher.useXattrs {
		hasher.storeHash(path, hash)
	}
	return hash, err
}

// storeHash stores the hash of a file on it as an xattr.
// This is best-effort since if it fails we can always fall back to a slower but reliable rehash.
func (hasher *PathHasher) storeHash(path string, hash []byte) {
	// Only ever store hashes on output files.
	if !strings.HasPrefix(path, "plz-out/") {
		return
	}
	if err := xattr.LSet(path, hasher.xattrName, hash); err != nil && os.IsPermission(err) {
		// If we get a permission denied, that may be because the output file was readonly.
		// Cheekily attempt to chmod it into submission.
		if info, err := os.Lstat(path); err == nil {
			if err := os.Chmod(path, info.Mode()|0220); err == nil {
				xattr.LSet(path, hasher.xattrName, hash)
				os.Chmod(path, info.Mode())
			}
		}
	}
}

// Calculate the hash of a single file
func (hasher *PathHasher) fileHash(h hash.Hash, filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	_, err = io.Copy(h, file)
	file.Close()
	return err
}

// ensureRelative ensures a path is relative to the repo root.
// This is important for getting best performance from memoizing the path hashes.
func (hasher *PathHasher) ensureRelative(path string) string {
	if strings.HasPrefix(path, hasher.root) {
		return strings.TrimLeft(strings.TrimPrefix(path, hasher.root), "/")
	}
	return path
}
<commit_msg>Reduce symlink checking to a warning for absolute paths (#968)<commit_after>package fs

import (
	"fmt"
	"hash"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/pkg/xattr"
)

// boolTrueHashValue is used when we need to write something indicating a bool in the input.
var boolTrueHashValue = []byte{2}

// A PathHasher is responsible for hashing & remembering paths.
type PathHasher struct {
	new       func() hash.Hash
	memo      map[string][]byte
	wait      map[string]*pendingHash
	mutex     sync.RWMutex
	root      string
	xattrName string
	useXattrs bool
}

type pendingHash struct {
	Ch   chan struct{}
	Hash []byte
	Err  error
}

// NewPathHasher returns a new PathHasher based on the given root directory.
func NewPathHasher(root string, useXattrs bool, hash func() hash.Hash, hashSuffix string) *PathHasher {
	return &PathHasher{
		new:       hash,
		memo:      map[string][]byte{},
		wait:      map[string]*pendingHash{},
		root:      root,
		useXattrs: useXattrs,
		xattrName: "user.plz_hash" + hashSuffix,
	}
}

// Size returns the size of the hash this hasher will return, in bytes.
func (hasher *PathHasher) Size() int {
	return hasher.new().Size()
}

// NewHash returns a new hash.Hash instance from this hasher.
func (hasher *PathHasher) NewHash() hash.Hash {
	return hasher.new()
}

// DisableXattrs turns off xattr support, which bypasses using them to record file hashes.
func (hasher *PathHasher) DisableXattrs() {
	hasher.useXattrs = false
}

// Hash hashes a single path.
// It is memoised and so will only hash each path once, unless recalc is true which will
// then force a recalculation of it.
// If store is true then the hash may be stored permanently; this should not be set for files that
// are user-controlled.
// TODO(peterebden): ensure that xattrs are marked correctly on cache retrieval.
func (hasher *PathHasher) Hash(path string, recalc, store bool) ([]byte, error) {
	path = hasher.ensureRelative(path)
	if !recalc {
		hasher.mutex.RLock()
		cached, present := hasher.memo[path]
		hasher.mutex.RUnlock()
		if present {
			return cached, nil
		}
	}
	// This check is important; if the file doesn't exist now, we don't want that
	// recorded forever in hasher.wait since it might get created later.
	if !PathExists(path) {
		return nil, fmt.Errorf("cannot calculate hash for %s: %s", path, os.ErrNotExist)
	}
	hasher.mutex.Lock()
	if pending, present := hasher.wait[path]; present {
		hasher.mutex.Unlock()
		<-pending.Ch
		return pending.Hash, pending.Err
	}
	pending := &pendingHash{Ch: make(chan struct{})}
	hasher.wait[path] = pending
	hasher.mutex.Unlock()
	result, err := hasher.hash(path, store, !recalc)
	hasher.mutex.Lock()
	if err == nil {
		hasher.memo[path] = result
	}
	delete(hasher.wait, path)
	hasher.mutex.Unlock()
	pending.Hash = result
	pending.Err = err
	close(pending.Ch)
	return result, err
}

// MustHash is as Hash but panics on error.
func (hasher *PathHasher) MustHash(path string) []byte {
	hash, err := hasher.Hash(path, false, false)
	if err != nil {
		panic(err)
	}
	return hash
}

// MoveHash is used when we move files from tmp to out and there was one there before; that's
// the only case in which the hash of a filepath could change.
func (hasher *PathHasher) MoveHash(oldPath, newPath string, copy bool) {
	oldPath = hasher.ensureRelative(oldPath)
	newPath = hasher.ensureRelative(newPath)
	hasher.mutex.Lock()
	defer hasher.mutex.Unlock()
	if oldHash, present := hasher.memo[oldPath]; present {
		hasher.memo[newPath] = oldHash
		// If the path is in plz-out/tmp we aren't ever going to use it again, so free some space.
		if !copy && strings.HasPrefix(oldPath, "plz-out/tmp") {
			delete(hasher.memo, oldPath)
		}
	}
}

// SetHash is used to directly set a hash for a path.
// This is used for remote files where we download them & therefore know the hash as they come in.
// TODO(peterebden): We should probably use this more for things like caches and so forth...
func (hasher *PathHasher) SetHash(path string, hash []byte) {
	hasher.mutex.Lock()
	hasher.memo[path] = hash
	hasher.mutex.Unlock()
	hasher.storeHash(path, hash)
}

func (hasher *PathHasher) hash(path string, store, read bool) ([]byte, error) {
	// Try to read xattrs first so we don't have to hash the whole thing.
	if read && strings.HasPrefix(path, "plz-out/") && hasher.useXattrs {
		if b, err := xattr.LGet(path, hasher.xattrName); err == nil {
			return b, nil
		}
	}
	h := hasher.new()
	info, err := os.Lstat(path)
	if err == nil && info.Mode()&os.ModeSymlink != 0 {
		// Handle symlinks specially (don't attempt to read their contents).
		dest, err := os.Readlink(path)
		if err != nil {
			return nil, err
		}
		// Write something arbitrary indicating this is a symlink.
		// This isn't quite perfect - it could potentially get mixed up with a file with the
		// appropriate contents, but that is not really likely.
		h.Write(boolTrueHashValue)
		if rel := hasher.ensureRelative(dest); (rel != dest || !filepath.IsAbs(dest)) && !filepath.IsAbs(path) {
			// Inside the root of our repo so it's something we manage - just hash its (relative) destination
			h.Write([]byte(rel))
		} else {
			// Outside the repo; it's a system tool, so we hash its contents.
			err := hasher.fileHash(h, path)
			return h.Sum(nil), err
		}
		return h.Sum(nil), nil
	} else if err == nil && info.IsDir() {
		err = WalkMode(path, func(p string, isDir bool, mode os.FileMode) error {
			if mode&os.ModeSymlink != 0 {
				// Is a symlink, must verify that it's not absolute.
				deref, err := os.Readlink(p)
				if err != nil {
					return err
				}
				if filepath.IsAbs(deref) {
					log.Warning("Symlink %s has an absolute target %s, that will likely be broken later")
				}
				// Deliberately do not attempt to read it. We will read the contents later since
				// it is a link within the temp dir anyway, and if it's a link to a directory
				// it can introduce a cycle.
				// Just write something to the hash indicating that we found something here,
				// otherwise rules might be marked as unchanged if they added additional symlinks.
				h.Write(boolTrueHashValue)
			} else if !isDir {
				return hasher.fileHash(h, p)
			}
			return nil
		})
	} else {
		err = hasher.fileHash(h, path) // let this handle any other errors
	}
	hash := h.Sum(nil)
	if err != nil {
		return hash, err
	} else if store && hasher.useXattrs {
		hasher.storeHash(path, hash)
	}
	return hash, err
}

// storeHash stores the hash of a file on it as an xattr.
// This is best-effort since if it fails we can always fall back to a slower but reliable rehash.
func (hasher *PathHasher) storeHash(path string, hash []byte) {
	// Only ever store hashes on output files.
	if !strings.HasPrefix(path, "plz-out/") {
		return
	}
	if err := xattr.LSet(path, hasher.xattrName, hash); err != nil && os.IsPermission(err) {
		// If we get a permission denied, that may be because the output file was readonly.
		// Cheekily attempt to chmod it into submission.
		if info, err := os.Lstat(path); err == nil {
			if err := os.Chmod(path, info.Mode()|0220); err == nil {
				xattr.LSet(path, hasher.xattrName, hash)
				os.Chmod(path, info.Mode())
			}
		}
	}
}

// Calculate the hash of a single file
func (hasher *PathHasher) fileHash(h hash.Hash, filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	_, err = io.Copy(h, file)
	file.Close()
	return err
}

// ensureRelative ensures a path is relative to the repo root.
// This is important for getting best performance from memoizing the path hashes.
func (hasher *PathHasher) ensureRelative(path string) string {
	if strings.HasPrefix(path, hasher.root) {
		return strings.TrimLeft(strings.TrimPrefix(path, hasher.root), "/")
	}
	return path
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager

import (
	"fmt"
	"reflect"
	"time"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
	"k8s.io/klog/v2"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
	"sigs.k8s.io/structured-merge-diff/v4/merge"
)

// DefaultMaxUpdateManagers defines the default maximum retained number of managedFields entries from updates
// if the number of update managers exceeds this, the oldest entries will be merged until the number is below the maximum.
// TODO(jennybuckley): Determine if this is really the best value. Ideally we wouldn't unnecessarily merge too many entries.
const DefaultMaxUpdateManagers int = 10

// DefaultTrackOnCreateProbability defines the default probability that the field management of an object
// starts being tracked from the object's creation, instead of from the first time the object is applied to.
const DefaultTrackOnCreateProbability float32 = 1

var atMostEverySecond = internal.NewAtMostEvery(time.Second)

// Managed groups a fieldpath.ManagedFields together with the timestamps associated with each operation.
type Managed interface {
	// Fields gets the fieldpath.ManagedFields.
	Fields() fieldpath.ManagedFields

	// Times gets the timestamps associated with each operation.
	Times() map[string]*metav1.Time
}

// Manager updates the managed fields and merges applied configurations.
type Manager interface {
	// Update is used when the object has already been merged (non-apply
	// use-case), and simply updates the managed fields in the output
	// object.
	//  * `liveObj` is not mutated by this function
	//  * `newObj` may be mutated by this function
	// Returns the new object with managedFields removed, and the object's new
	// proposed managedFields separately.
	Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error)

	// Apply is used when server-side apply is called, as it merges the
	// object and updates the managed fields.
	//  * `liveObj` is not mutated by this function
	//  * `newObj` may be mutated by this function
	// Returns the new object with managedFields removed, and the object's new
	// proposed managedFields separately.
	Apply(liveObj, appliedObj runtime.Object, managed Managed, fieldManager string, force bool) (runtime.Object, Managed, error)
}

// FieldManager updates the managed fields and merge applied
// configurations.
type FieldManager struct {
	fieldManager Manager
	subresource  string
}

// NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
// on update and apply requests.
func NewFieldManager(f Manager, subresource string) *FieldManager {
	return &FieldManager{fieldManager: f, subresource: subresource}
}

// NewDefaultFieldManager creates a new FieldManager that merges apply requests
// and update managed fields for other types of requests.
func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (*FieldManager, error) {
	f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
	if err != nil {
		return nil, fmt.Errorf("failed to create field manager: %v", err)
	}
	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
}

// NewDefaultCRDFieldManager creates a new FieldManager specifically for
// CRDs. This allows for the possibility of fields which are not defined
// in models, as well as having no models defined at all.
func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (_ *FieldManager, err error) {
	f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
	if err != nil {
		return nil, fmt.Errorf("failed to create field manager: %v", err)
	}
	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
}

// newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, subresource string) *FieldManager {
	return NewFieldManager(
		NewLastAppliedUpdater(
			NewLastAppliedManager(
				NewProbabilisticSkipNonAppliedManager(
					NewCapManagersManager(
						NewBuildManagerInfoManager(
							NewManagedFieldsUpdater(
								NewStripMetaManager(f),
							), kind.GroupVersion(), subresource,
						), DefaultMaxUpdateManagers,
					), objectCreater, kind, DefaultTrackOnCreateProbability,
				), typeConverter, objectConverter, kind.GroupVersion()),
		), subresource,
	)
}

// DecodeManagedFields converts ManagedFields from the wire format (api format)
// to the format used by sigs.k8s.io/structured-merge-diff
func DecodeManagedFields(encodedManagedFields []metav1.ManagedFieldsEntry) (Managed, error) {
	return internal.DecodeManagedFields(encodedManagedFields)
}

func decodeLiveOrNew(liveObj, newObj runtime.Object, ignoreManagedFieldsFromRequestObject bool) (Managed, error) {
	liveAccessor, err := meta.Accessor(liveObj)
	if err != nil {
		return nil, err
	}

	// We take the managedFields of the live object in case the request tries to
	// manually set managedFields via a subresource.
	if ignoreManagedFieldsFromRequestObject {
		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
	}

	// If the object doesn't have metadata, we should just return without trying to
	// set the managedFields at all, so creates/updates/patches will work normally.
	newAccessor, err := meta.Accessor(newObj)
	if err != nil {
		return nil, err
	}

	if isResetManagedFields(newAccessor.GetManagedFields()) {
		return internal.NewEmptyManaged(), nil
	}

	// If the managed field is empty or we failed to decode it,
	// let's try the live object. This is to prevent clients who
	// don't understand managedFields from deleting it accidentally.
	managed, err := DecodeManagedFields(newAccessor.GetManagedFields())
	if err != nil || len(managed.Fields()) == 0 {
		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
	}
	return managed, nil
}

func emptyManagedFieldsOnErr(managed Managed, err error) (Managed, error) {
	if err != nil {
		return internal.NewEmptyManaged(), nil
	}
	return managed, nil
}

// Update is used when the object has already been merged (non-apply
// use-case), and simply updates the managed fields in the output
// object.
func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
	// First try to decode the managed fields provided in the update,
	// This is necessary to allow directly updating managed fields.
	isSubresource := f.subresource != ""
	managed, err := decodeLiveOrNew(liveObj, newObj, isSubresource)
	if err != nil {
		return newObj, nil
	}

	internal.RemoveObjectManagedFields(newObj)

	if object, managed, err = f.fieldManager.Update(liveObj, newObj, managed, manager); err != nil {
		return nil, err
	}

	if err = internal.EncodeObjectManagedFields(object, managed); err != nil {
		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
	}

	return object, nil
}

// UpdateNoErrors is the same as Update, but it will not return
// errors. If an error happens, the object is returned with
// managedFields cleared.
func (f *FieldManager) UpdateNoErrors(liveObj, newObj runtime.Object, manager string) runtime.Object {
	obj, err := f.Update(liveObj, newObj, manager)
	if err != nil {
		atMostEverySecond.Do(func() {
			ns, name := "unknown", "unknown"
			accessor, err := meta.Accessor(newObj)
			if err == nil {
				ns = accessor.GetNamespace()
				name = accessor.GetName()
			}

			klog.ErrorS(err, "[SHOULD NOT HAPPEN] failed to update managedFields", "VersionKind",
				newObj.GetObjectKind().GroupVersionKind(), "namespace", ns, "name", name)
		})
		// Explicitly remove managedFields on failure, so that
		// we can't have garbage in it.
		internal.RemoveObjectManagedFields(newObj)
		return newObj
	}
	return obj
}

// Returns true if the managedFields indicate that the user is trying to
// reset the managedFields, i.e. if the list is non-nil but empty, or if
// the list has one empty item.
func isResetManagedFields(managedFields []metav1.ManagedFieldsEntry) bool {
	if len(managedFields) == 0 {
		return managedFields != nil
	}

	if len(managedFields) == 1 {
		return reflect.DeepEqual(managedFields[0], metav1.ManagedFieldsEntry{})
	}

	return false
}

// Apply is used when server-side apply is called, as it merges the
// object and updates the managed fields.
func (f *FieldManager) Apply(liveObj, appliedObj runtime.Object, manager string, force bool) (object runtime.Object, err error) {
	// If the object doesn't have metadata, apply isn't allowed.
	accessor, err := meta.Accessor(liveObj)
	if err != nil {
		return nil, fmt.Errorf("couldn't get accessor: %v", err)
	}

	// Decode the managed fields in the live object, since it isn't allowed in the patch.
	managed, err := DecodeManagedFields(accessor.GetManagedFields())
	if err != nil {
		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
	}

	object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force)
	if err != nil {
		if conflicts, ok := err.(merge.Conflicts); ok {
			return nil, internal.NewConflictError(conflicts)
		}
		return nil, err
	}

	if err = internal.EncodeObjectManagedFields(object, managed); err != nil {
		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
	}

	return object, nil
}
<commit_msg>apiserver: use the correct error when logging errors updating managedFields<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager

import (
	"fmt"
	"reflect"
	"time"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
	"k8s.io/klog/v2"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
	"sigs.k8s.io/structured-merge-diff/v4/merge"
)

// DefaultMaxUpdateManagers defines the default maximum retained number of managedFields entries from updates
// if the number of update managers exceeds this, the oldest entries will be merged until the number is below the maximum.
// TODO(jennybuckley): Determine if this is really the best value. Ideally we wouldn't unnecessarily merge too many entries.
const DefaultMaxUpdateManagers int = 10

// DefaultTrackOnCreateProbability defines the default probability that the field management of an object
// starts being tracked from the object's creation, instead of from the first time the object is applied to.
const DefaultTrackOnCreateProbability float32 = 1

var atMostEverySecond = internal.NewAtMostEvery(time.Second)

// Managed groups a fieldpath.ManagedFields together with the timestamps associated with each operation.
type Managed interface {
	// Fields gets the fieldpath.ManagedFields.
	Fields() fieldpath.ManagedFields

	// Times gets the timestamps associated with each operation.
	Times() map[string]*metav1.Time
}

// Manager updates the managed fields and merges applied configurations.
type Manager interface {
	// Update is used when the object has already been merged (non-apply
	// use-case), and simply updates the managed fields in the output
	// object.
	//  * `liveObj` is not mutated by this function
	//  * `newObj` may be mutated by this function
	// Returns the new object with managedFields removed, and the object's new
	// proposed managedFields separately.
	Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error)

	// Apply is used when server-side apply is called, as it merges the
	// object and updates the managed fields.
	//  * `liveObj` is not mutated by this function
	//  * `newObj` may be mutated by this function
	// Returns the new object with managedFields removed, and the object's new
	// proposed managedFields separately.
	Apply(liveObj, appliedObj runtime.Object, managed Managed, fieldManager string, force bool) (runtime.Object, Managed, error)
}

// FieldManager updates the managed fields and merge applied
// configurations.
type FieldManager struct {
	fieldManager Manager
	subresource  string
}

// NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
// on update and apply requests.
func NewFieldManager(f Manager, subresource string) *FieldManager {
	return &FieldManager{fieldManager: f, subresource: subresource}
}

// NewDefaultFieldManager creates a new FieldManager that merges apply requests
// and update managed fields for other types of requests.
func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (*FieldManager, error) {
	f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
	if err != nil {
		return nil, fmt.Errorf("failed to create field manager: %v", err)
	}
	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
}

// NewDefaultCRDFieldManager creates a new FieldManager specifically for
// CRDs. This allows for the possibility of fields which are not defined
// in models, as well as having no models defined at all.
func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (_ *FieldManager, err error) {
	f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
	if err != nil {
		return nil, fmt.Errorf("failed to create field manager: %v", err)
	}
	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
}

// newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, subresource string) *FieldManager {
	return NewFieldManager(
		NewLastAppliedUpdater(
			NewLastAppliedManager(
				NewProbabilisticSkipNonAppliedManager(
					NewCapManagersManager(
						NewBuildManagerInfoManager(
							NewManagedFieldsUpdater(
								NewStripMetaManager(f),
							), kind.GroupVersion(), subresource,
						), DefaultMaxUpdateManagers,
					), objectCreater, kind, DefaultTrackOnCreateProbability,
				), typeConverter, objectConverter, kind.GroupVersion()),
		), subresource,
	)
}

// DecodeManagedFields converts ManagedFields from the wire format (api format)
// to the format used by sigs.k8s.io/structured-merge-diff
func DecodeManagedFields(encodedManagedFields []metav1.ManagedFieldsEntry) (Managed, error) {
	return internal.DecodeManagedFields(encodedManagedFields)
}

func decodeLiveOrNew(liveObj, newObj runtime.Object, ignoreManagedFieldsFromRequestObject bool) (Managed, error) {
	liveAccessor, err := meta.Accessor(liveObj)
	if err != nil {
		return nil, err
	}

	// We take the managedFields of the live object in case the request tries to
	// manually set managedFields via a subresource.
	if ignoreManagedFieldsFromRequestObject {
		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
	}

	// If the object doesn't have metadata, we should just return without trying to
	// set the managedFields at all, so creates/updates/patches will work normally.
	newAccessor, err := meta.Accessor(newObj)
	if err != nil {
		return nil, err
	}

	if isResetManagedFields(newAccessor.GetManagedFields()) {
		return internal.NewEmptyManaged(), nil
	}

	// If the managed field is empty or we failed to decode it,
	// let's try the live object. This is to prevent clients who
	// don't understand managedFields from deleting it accidentally.
	managed, err := DecodeManagedFields(newAccessor.GetManagedFields())
	if err != nil || len(managed.Fields()) == 0 {
		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
	}
	return managed, nil
}

func emptyManagedFieldsOnErr(managed Managed, err error) (Managed, error) {
	if err != nil {
		return internal.NewEmptyManaged(), nil
	}
	return managed, nil
}

// Update is used when the object has already been merged (non-apply
// use-case), and simply updates the managed fields in the output
// object.
func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
	// First try to decode the managed fields provided in the update,
	// This is necessary to allow directly updating managed fields.
	isSubresource := f.subresource != ""
	managed, err := decodeLiveOrNew(liveObj, newObj, isSubresource)
	if err != nil {
		return newObj, nil
	}

	internal.RemoveObjectManagedFields(newObj)

	if object, managed, err = f.fieldManager.Update(liveObj, newObj, managed, manager); err != nil {
		return nil, err
	}

	if err = internal.EncodeObjectManagedFields(object, managed); err != nil {
		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
	}

	return object, nil
}

// UpdateNoErrors is the same as Update, but it will not return
// errors. If an error happens, the object is returned with
// managedFields cleared.
func (f *FieldManager) UpdateNoErrors(liveObj, newObj runtime.Object, manager string) runtime.Object {
	obj, err := f.Update(liveObj, newObj, manager)
	if err != nil {
		atMostEverySecond.Do(func() {
			ns, name := "unknown", "unknown"
			if accessor, err := meta.Accessor(newObj); err == nil {
				ns = accessor.GetNamespace()
				name = accessor.GetName()
			}

			klog.ErrorS(err, "[SHOULD NOT HAPPEN] failed to update managedFields", "VersionKind",
				newObj.GetObjectKind().GroupVersionKind(), "namespace", ns, "name", name)
		})
		// Explicitly remove managedFields on failure, so that
		// we can't have garbage in it.
		internal.RemoveObjectManagedFields(newObj)
		return newObj
	}
	return obj
}

// Returns true if the managedFields indicate that the user is trying to
// reset the managedFields, i.e. if the list is non-nil but empty, or if
// the list has one empty item.
func isResetManagedFields(managedFields []metav1.ManagedFieldsEntry) bool {
	if len(managedFields) == 0 {
		return managedFields != nil
	}

	if len(managedFields) == 1 {
		return reflect.DeepEqual(managedFields[0], metav1.ManagedFieldsEntry{})
	}

	return false
}

// Apply is used when server-side apply is called, as it merges the
// object and updates the managed fields.
func (f *FieldManager) Apply(liveObj, appliedObj runtime.Object, manager string, force bool) (object runtime.Object, err error) {
	// If the object doesn't have metadata, apply isn't allowed.
	accessor, err := meta.Accessor(liveObj)
	if err != nil {
		return nil, fmt.Errorf("couldn't get accessor: %v", err)
	}

	// Decode the managed fields in the live object, since it isn't allowed in the patch.
	managed, err := DecodeManagedFields(accessor.GetManagedFields())
	if err != nil {
		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
	}

	object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force)
	if err != nil {
		if conflicts, ok := err.(merge.Conflicts); ok {
			return nil, internal.NewConflictError(conflicts)
		}
		return nil, err
	}

	if err = internal.EncodeObjectManagedFields(object, managed); err != nil {
		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
	}

	return object, nil
}
<|endoftext|>


<commit_before>package notification

import (
	"bytes"
	"fmt"
	"github.com/fritzpay/paymentd/pkg/paymentd/payment"
	paymentService "github.com/fritzpay/paymentd/pkg/service/payment"
)

const (
	PaymentNotificationVersion = "2.0"
)

type PaymentNotification struct {
	Version              string
	PaymentId            payment.PaymentID
	Ident                string
	Amount               int64 `json:",string"`
	Subunits             int8  `json:",string"`
	DecimalAmount        string
	Currency             string
	Country              string          `json:",omitempty"`
	PaymentMethodId      int64           `json:",string,omitempty"`
	Locale               string          `json:",omitempty"`
	Balance              payment.Balance `json:",omitempty"`
	Status               string          `json:",omitempty"`
	TransactionTimestamp int64           `json:",string,omitempty"`
	Metadata             map[string]string
	Timestamp            int64  `json:",string"`
	Nonce                string `json:",omitempty"`
	Signature            string `json:",omitempty"`
}

func NewPaymentNotification(srv *paymentService.Service, p *payment.Payment) (*PaymentNotification, error) {
	n := &PaymentNotification{
		Version:       PaymentNotificationVersion,
		PaymentId:     srv.EncodedPaymentID(p.PaymentID()),
		Ident:         p.Ident,
		Amount:        p.Amount,
		Subunits:      p.Subunits,
		DecimalAmount: p.Decimal().String(),
		Currency:      p.Currency,
		Metadata:      p.Metadata,
	}
	if !p.Config.IsConfigured() {
		return n, nil
	}
	if p.Config.Country.Valid {
		n.Country = p.Config.Country.String
	}
	if p.Config.PaymentMethodID.Valid {
		n.PaymentMethodId = p.Config.PaymentMethodID.Int64
	}
	if p.Config.Locale.Valid {
		n.Locale = p.Config.Locale.String
	}
	return n, nil
}

func (n *PaymentNotification) Message() ([]byte, error) {
	var err error
	buf := bytes.NewBuffer(nil)
	_, err = buf.WriteString(n.Version)
	if err != nil {
		return nil, fmt.Errorf("buffer write error: %v", err)
	}
	return buf.Bytes(), nil
}
<commit_msg>denote payment notification v2 as alpha<commit_after>package notification

import (
	"bytes"
	"fmt"
	"github.com/fritzpay/paymentd/pkg/paymentd/payment"
	paymentService "github.com/fritzpay/paymentd/pkg/service/payment"
)

const (
	PaymentNotificationVersion = "2.0.0-alpha"
)

type PaymentNotification struct {
	Version              string
	PaymentId            payment.PaymentID
	Ident                string
	Amount               int64 `json:",string"`
	Subunits             int8  `json:",string"`
	DecimalAmount        string
	Currency             string
	Country              string          `json:",omitempty"`
	PaymentMethodId      int64           `json:",string,omitempty"`
	Locale               string          `json:",omitempty"`
	Balance              payment.Balance `json:",omitempty"`
	Status               string          `json:",omitempty"`
	TransactionTimestamp int64           `json:",string,omitempty"`
	Metadata             map[string]string
	Timestamp            int64  `json:",string"`
	Nonce                string `json:",omitempty"`
	Signature            string `json:",omitempty"`
}

func NewPaymentNotification(srv *paymentService.Service, p *payment.Payment) (*PaymentNotification, error) {
	n := &PaymentNotification{
		Version:       PaymentNotificationVersion,
		PaymentId:     srv.EncodedPaymentID(p.PaymentID()),
		Ident:         p.Ident,
		Amount:        p.Amount,
		Subunits:      p.Subunits,
		DecimalAmount: p.Decimal().String(),
		Currency:      p.Currency,
		Metadata:      p.Metadata,
	}
	if !p.Config.IsConfigured() {
		return n, nil
	}
	if p.Config.Country.Valid {
		n.Country = p.Config.Country.String
	}
	if p.Config.PaymentMethodID.Valid {
		n.PaymentMethodId = p.Config.PaymentMethodID.Int64
	}
	if p.Config.Locale.Valid {
		n.Locale = p.Config.Locale.String
	}
	return n, nil
}

func (n *PaymentNotification) Message() ([]byte, error) {
	var err error
	buf := bytes.NewBuffer(nil)
	_, err = buf.WriteString(n.Version)
	if err != nil {
		return nil, fmt.Errorf("buffer write error: %v", err)
	}
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

var (
	copy CopyOptions
)

type CopyOptions struct {
	filerGrpcPort  *int
	master         *string
	include        *string
	replication    *string
	collection     *string
	ttl            *string
	maxMB          *int
	grpcDialOption grpc.DialOption
	masterClient   *wdclient.MasterClient
}

func init() {
	cmdCopy.Run = runCopy // break init cycle
	cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
	copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
	copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
}

var cmdCopy = &Command{
	UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
	Short:     "copy one or a list of files to a filer folder",
	Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder

  It can copy one or a list of files or folders.

  If copying a whole folder recursively:
  All files under the folder and subfolders will be copyed.
  Optional parameter "-include" allows you to specify the file name patterns.

  If "maxMB" is set to a positive number, files larger than it would be split into chunks.

  `,
}

func runCopy(cmd *Command, args []string) bool {

	weed_server.LoadConfiguration("security", false)

	if len(args) <= 1 {
		return false
	}
	filerDestination := args[len(args)-1]
	fileOrDirs := args[0 : len(args)-1]

	filerUrl, err := url.Parse(filerDestination)
	if err != nil {
		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
		return false
	}
	urlPath := filerUrl.Path
	if !strings.HasSuffix(urlPath, "/") {
		fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err)
		return false
	}

	if filerUrl.Port() == "" {
		fmt.Printf("The filer port should be specified.\n")
		return false
	}

	filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64)
	if parseErr != nil {
		fmt.Printf("The filer port parse error: %v\n", parseErr)
		return false
	}

	filerGrpcPort := filerPort + 10000
	if *copy.filerGrpcPort != 0 {
		filerGrpcPort = uint64(*copy.filerGrpcPort)
	}

	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")

	copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
	go copy.masterClient.KeepConnectedToMaster()
	copy.masterClient.WaitUntilConnected()

	for _, fileOrDir := range fileOrDirs {
		if !doEachCopy(context.Background(), fileOrDir, filerUrl.Host, filerGrpcAddress, copy.grpcDialOption, urlPath) {
			return false
		}
	}
	return true
}

func doEachCopy(ctx context.Context, fileOrDir string, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, path string) bool {
	f, err := os.Open(fileOrDir)
	if err != nil {
		fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
		if _, ok := err.(*os.PathError); ok {
			fmt.Printf("skipping %s\n", fileOrDir)
			return true
		}
		return false
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err)
		return false
	}

	mode := fi.Mode()
	if mode.IsDir() {
		files, _ := ioutil.ReadDir(fileOrDir)
		for _, subFileOrDir := range files {
			if !doEachCopy(ctx, fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, grpcDialOption, path+fi.Name()+"/") {
				return false
			}
		}
		return true
	}

	// this is a regular file
	if *copy.include != "" {
		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
			return true
		}
	}

	// find the chunk count
	chunkSize := int64(*copy.maxMB * 1024 * 1024)
	chunkCount := 1
	if chunkSize > 0 && fi.Size() > chunkSize {
		chunkCount = int(fi.Size()/chunkSize) + 1
	}

	if chunkCount == 1 {
		return uploadFileAsOne(ctx, filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi)
	}

	return uploadFileInChunks(ctx, filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi, chunkCount, chunkSize)
}

func uploadFileAsOne(ctx context.Context, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo) bool {

	// upload the file content
	fileName := filepath.Base(f.Name())
	mimeType := detectMimeType(f)

	var chunks []*filer_pb.FileChunk

	if fi.Size() > 0 {

		// assign a volume
		assignResult, err := operation.Assign(copy.masterClient.GetMaster(), grpcDialOption, &operation.VolumeAssignRequest{
			Count:       1,
			Replication: *copy.replication,
			Collection:  *copy.collection,
			Ttl:         *copy.ttl,
		})
		if err != nil {
			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
		}

		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid

		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
		if err != nil {
			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
			return false
		}
		if uploadResult.Error != "" {
			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
			return false
		}
		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)

		chunks = append(chunks, &filer_pb.FileChunk{
			FileId: assignResult.Fid,
			Offset: 0,
			Size:   uint64(uploadResult.Size),
			Mtime:  time.Now().UnixNano(),
			ETag:   uploadResult.ETag,
		})

		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
	}

	if err := withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		request := &filer_pb.CreateEntryRequest{
			Directory: urlFolder,
			Entry: &filer_pb.Entry{
				Name: fileName,
				Attributes: &filer_pb.FuseAttributes{
					Crtime:      time.Now().Unix(),
					Mtime:       time.Now().Unix(),
					Gid:         uint32(os.Getgid()),
					Uid:         uint32(os.Getuid()),
					FileSize:    uint64(fi.Size()),
					FileMode:    uint32(fi.Mode()),
					Mime:        mimeType,
					Replication: *copy.replication,
					Collection:  *copy.collection,
					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
				},
				Chunks: chunks,
			},
		}

		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("update fh: %v", err)
		}
		return nil
	}); err != nil {
		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
		return false
	}

	return true
}

func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {

	fileName := filepath.Base(f.Name())
	mimeType := detectMimeType(f)

	var chunks []*filer_pb.FileChunk

	for i := int64(0); i < int64(chunkCount); i++ {

		// assign a volume
		assignResult, err := operation.Assign(copy.masterClient.GetMaster(), grpcDialOption, &operation.VolumeAssignRequest{
			Count:       1,
			Replication: *copy.replication,
			Collection:  *copy.collection,
			Ttl:         *copy.ttl,
		})
		if err != nil {
			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
		}

		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid

		uploadResult, err := operation.Upload(targetUrl,
			fileName+"-"+strconv.FormatInt(i+1, 10),
			io.LimitReader(f, chunkSize),
			false, "application/octet-stream", nil, assignResult.Auth)
		if err != nil {
			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
			return false
		}
		if uploadResult.Error != "" {
			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
			return false
		}
		chunks = append(chunks, &filer_pb.FileChunk{
			FileId: assignResult.Fid,
			Offset: i * chunkSize,
			Size:   uint64(uploadResult.Size),
			Mtime:  time.Now().UnixNano(),
			ETag:   uploadResult.ETag,
		})
		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
	}

	if err := withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		request := &filer_pb.CreateEntryRequest{
			Directory: urlFolder,
			Entry: &filer_pb.Entry{
				Name: fileName,
				Attributes: &filer_pb.FuseAttributes{
					Crtime:      time.Now().Unix(),
					Mtime:       time.Now().Unix(),
					Gid:         uint32(os.Getgid()),
					Uid:         uint32(os.Getuid()),
					FileSize:    uint64(fi.Size()),
					FileMode:    uint32(fi.Mode()),
					Mime:        mimeType,
					Replication: *copy.replication,
					Collection:  *copy.collection,
					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
				},
				Chunks: chunks,
			},
		}

		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("update fh: %v", err)
		}
		return nil
	}); err != nil {
		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
		return false
	}

	fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)

	return true
}

func detectMimeType(f *os.File) string {
	head := make([]byte, 512)
	f.Seek(0, io.SeekStart)
	n, err := f.Read(head)
	if err == io.EOF {
		return ""
	}
	if err != nil {
		fmt.Printf("read head of %v: %v\n", f.Name(), err)
		return "application/octet-stream"
	}
	f.Seek(0, io.SeekStart)
	mimeType := http.DetectContentType(head[:n])
	return mimeType
}

func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {

	grpcConnection, err := util.GrpcDial(ctx, filerAddress, grpcDialOption)
	if err != nil {
		return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
	}
	defer grpcConnection.Close()

	client := filer_pb.NewSeaweedFilerClient(grpcConnection)

	return fn(client)
}
<commit_msg>weed filer.copy: use cached filer grpc connection<commit_after>package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

var (
	copy CopyOptions
)

type CopyOptions struct {
	filerGrpcPort  *int
	master         *string
	include        *string
	replication    *string
	collection     *string
	ttl            *string
	maxMB          *int
	grpcDialOption grpc.DialOption
	masterClient   *wdclient.MasterClient
}

func init() {
	cmdCopy.Run = runCopy // break init cycle
	cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
	copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
	copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
}

var cmdCopy = &Command{
	UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
	Short:     "copy one or a list of files to a filer folder",
	Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder

  It can copy one or a list of files or folders.

  If copying a whole folder recursively:
  All files under the folder and subfolders will be copyed.
  Optional parameter "-include" allows you to specify the file name patterns.

  If "maxMB" is set to a positive number, files larger than it would be split into chunks.

  `,
}

func runCopy(cmd *Command, args []string) bool {

	weed_server.LoadConfiguration("security", false)

	if len(args) <= 1 {
		return false
	}
	filerDestination := args[len(args)-1]
	fileOrDirs := args[0 : len(args)-1]

	filerUrl, err := url.Parse(filerDestination)
	if err != nil {
		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
		return false
	}
	urlPath := filerUrl.Path
	if !strings.HasSuffix(urlPath, "/") {
		fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err)
		return false
	}

	if filerUrl.Port() == "" {
		fmt.Printf("The filer port should be specified.\n")
		return false
	}

	filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64)
	if parseErr != nil {
		fmt.Printf("The filer port parse error: %v\n", parseErr)
		return false
	}

	filerGrpcPort := filerPort + 10000
	if *copy.filerGrpcPort != 0 {
		filerGrpcPort = uint64(*copy.filerGrpcPort)
	}

	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")

	copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
	go copy.masterClient.KeepConnectedToMaster()
	copy.masterClient.WaitUntilConnected()

	for _, fileOrDir := range fileOrDirs {
		if !doEachCopy(context.Background(), fileOrDir, filerUrl.Host, filerGrpcAddress, copy.grpcDialOption, urlPath) {
			return false
		}
	}
	return true
}

func doEachCopy(ctx context.Context, fileOrDir string, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, path string) bool {
	f, err := os.Open(fileOrDir)
	if err != nil {
		fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
		if _, ok := err.(*os.PathError); ok {
			fmt.Printf("skipping %s\n", fileOrDir)
			return true
		}
		return false
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err)
		return false
	}

	mode := fi.Mode()
	if mode.IsDir() {
		files, _ := ioutil.ReadDir(fileOrDir)
		for _, subFileOrDir := range files {
			if !doEachCopy(ctx, fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, grpcDialOption, path+fi.Name()+"/") {
				return false
			}
		}
		return true
	}

	// this is a regular file
	if *copy.include != "" {
		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
			return true
		}
	}

	// find the chunk count
	chunkSize := int64(*copy.maxMB * 1024 * 1024)
	chunkCount := 1
	if chunkSize > 0 && fi.Size() > chunkSize {
		chunkCount = int(fi.Size()/chunkSize) + 1
	}

	if chunkCount == 1 {
		return uploadFileAsOne(ctx, filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi)
	}

	return uploadFileInChunks(ctx, filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi, chunkCount, chunkSize)
}

func uploadFileAsOne(ctx context.Context, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo) bool {

	// upload the file content
	fileName := filepath.Base(f.Name())
	mimeType := detectMimeType(f)

	var chunks []*filer_pb.FileChunk

	if fi.Size() > 0 {

		// assign a volume
		assignResult, err := operation.Assign(copy.masterClient.GetMaster(), grpcDialOption, &operation.VolumeAssignRequest{
			Count:       1,
			Replication: *copy.replication,
			Collection:  *copy.collection,
			Ttl:         *copy.ttl,
		})
		if err != nil {
			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
		}

		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid

		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
		if err != nil {
			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
			return false
		}
		if uploadResult.Error != "" {
			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
			return false
		}
		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)

		chunks = append(chunks, &filer_pb.FileChunk{
			FileId: assignResult.Fid,
			Offset: 0,
			Size:   uint64(uploadResult.Size),
			Mtime:  time.Now().UnixNano(),
			ETag:   uploadResult.ETag,
		})

		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
	}

	if err := withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		request := &filer_pb.CreateEntryRequest{
			Directory: urlFolder,
			Entry: &filer_pb.Entry{
				Name: fileName,
				Attributes: &filer_pb.FuseAttributes{
					Crtime:      time.Now().Unix(),
					Mtime:       time.Now().Unix(),
					Gid:         uint32(os.Getgid()),
					Uid:         uint32(os.Getuid()),
					FileSize:    uint64(fi.Size()),
					FileMode:    uint32(fi.Mode()),
					Mime:        mimeType,
					Replication: *copy.replication,
					Collection:  *copy.collection,
					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
				},
				Chunks: chunks,
			},
		}

		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("update fh: %v", err)
		}
		return nil
	}); err != nil {
		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
		return false
	}

	return true
}

func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {

	fileName := filepath.Base(f.Name())
	mimeType := detectMimeType(f)

	var chunks []*filer_pb.FileChunk

	for i := int64(0); i < int64(chunkCount); i++ {

		// assign a volume
		assignResult, err := operation.Assign(copy.masterClient.GetMaster(), grpcDialOption, &operation.VolumeAssignRequest{
			Count:       1,
			Replication: *copy.replication,
			Collection:  *copy.collection,
			Ttl:         *copy.ttl,
		})
		if err != nil {
			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
		}

		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid

		uploadResult, err := operation.Upload(targetUrl,
			fileName+"-"+strconv.FormatInt(i+1, 10),
			io.LimitReader(f, chunkSize),
			false, "application/octet-stream", nil, assignResult.Auth)
		if err != nil {
			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
			return false
		}
		if uploadResult.Error != "" {
			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
			return false
		}
		chunks = append(chunks, &filer_pb.FileChunk{
			FileId: assignResult.Fid,
			Offset: i * chunkSize,
			Size:   uint64(uploadResult.Size),
			Mtime:  time.Now().UnixNano(),
			ETag:   uploadResult.ETag,
		})
		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
	}

	if err := withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		request := &filer_pb.CreateEntryRequest{
			Directory: urlFolder,
			Entry: &filer_pb.Entry{
				Name: fileName,
				Attributes: &filer_pb.FuseAttributes{
					Crtime:      time.Now().Unix(),
					Mtime:       time.Now().Unix(),
					Gid:         uint32(os.Getgid()),
					Uid:         uint32(os.Getuid()),
					FileSize:    uint64(fi.Size()),
					FileMode:    uint32(fi.Mode()),
					Mime:        mimeType,
					Replication: *copy.replication,
					Collection:  *copy.collection,
					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
				},
				Chunks: chunks,
			},
		}

		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("update fh: %v", err)
		}
		return nil
	}); err != nil {
		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
		return false
	}

	fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)

	return true
}

func detectMimeType(f *os.File) string {
	head := make([]byte, 512)
	f.Seek(0, io.SeekStart)
	n, err := f.Read(head)
	if err == io.EOF {
		return ""
	}
	if err != nil {
		fmt.Printf("read head of %v: %v\n", f.Name(), err)
		return "application/octet-stream"
	}
	f.Seek(0, io.SeekStart)
	mimeType := http.DetectContentType(head[:n])
	return mimeType
}

func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {

	return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(clientConn)
		return fn(client)
	}, filerAddress, grpcDialOption)

}
<|endoftext|>


<commit_before>package mem

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"sync"
	"sync/atomic"
)

var pools []*sync.Pool

const (
	min_size = 1024
)

func bitCount(size int) (count int) {
	for ; size > min_size; count++ {
		size = (size + 1) >> 1
	}
	return
}

func init() {
	// 1KB ~ 256MB
	pools = make([]*sync.Pool, bitCount(1024*1024*256))
	for i := 0; i < len(pools); i++ {
		slotSize := 1024 << i
		pools[i] = &sync.Pool{
			New: func() interface{} {
				buffer := make([]byte, slotSize)
				return &buffer
			},
		}
	}
}

func getSlotPool(size int) *sync.Pool {
	index := bitCount(size)
	return pools[index]
}

var total int64

func Allocate(size int) []byte {
	pool := getSlotPool(size)
	if pool != nil {

		newVal := atomic.AddInt64(&total, 1)
		glog.V(4).Infof("++> %d", newVal)

		slab := *pool.Get().(*[]byte)
		return slab[:size]
	}
	return make([]byte, size)
}

func Free(buf []byte) {
	pool := getSlotPool(cap(buf))
	if pool != nil {
		newVal := atomic.AddInt64(&total, -1)
		glog.V(4).Infof("--> %d", newVal)
		pool.Put(&buf)
	}
}
<commit_msg>avoid pool memory allocation if too large<commit_after>package mem

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"sync"
	"sync/atomic"
)

var pools []*sync.Pool

const (
	min_size = 1024
)

func bitCount(size int) (count int) {
	for ; size > min_size; count++ {
		size = (size + 1) >> 1
	}
	return
}

func init() {
	// 1KB ~ 256MB
	pools = make([]*sync.Pool, bitCount(1024*1024*256))
	for i := 0; i < len(pools); i++ {
		slotSize := 1024 << i
		pools[i] = &sync.Pool{
			New: func() interface{} {
				buffer := make([]byte, slotSize)
				return &buffer
			},
		}
	}
}

func getSlotPool(size int) (*sync.Pool, bool) {
	index := bitCount(size)
	if index >= len(pools) {
		return nil, false
	}
	return pools[index], true
}

var total int64

func Allocate(size int) []byte {
	if pool, found := getSlotPool(size); found {
		newVal := atomic.AddInt64(&total, 1)
		glog.V(4).Infof("++> %d", newVal)

		slab := *pool.Get().(*[]byte)
		return slab[:size]
	}
	return make([]byte, size)
}

func Free(buf []byte) {
	if pool, found := getSlotPool(cap(buf)); found {
		newVal := atomic.AddInt64(&total, -1)
		glog.V(4).Infof("--> %d", newVal)
		pool.Put(&buf)
	}
}
<|endoftext|>