<commit_before>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

const page = `

    <title>go-hep/groot file inspector</title>
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" />
	<link rel="stylesheet" href="https://www.w3schools.com/w3css/3/w3.css">

	<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.7/themes/default/style.min.css" />

go-hep/groot ROOT file inspector

	{{if .Local}}

		File: <input id="groot-open-form-input" type="text" name="uri" value placeholder="URI to local or remote file">

		 Open

		<input id="groot-open-button" type="hidden" value="Open" onclick="openROOTFile()">

	{{- end}}

	<form id="groot-upload-form" enctype="multipart/form-data" action="/root-file-upload" method="post">

		 Upload

		<input id="groot-file-upload" type="file" name="groot-file"/>
		<input type="hidden" name="token" value="{{.Token}}"/>
		<input type="hidden" value="upload" />
	</form>

`
<commit_msg>groot/cmd/root-srv: add a spinner widget to indicate processing is taking place<commit_after>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

const page = `

    <title>go-hep/groot file inspector</title>
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" />
	<link rel="stylesheet" href="https://www.w3schools.com/w3css/3/w3.css">

	<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/jstree/3.3.7/themes/default/style.min.css" />

go-hep/groot ROOT file inspector

	{{if .Local}}

		File: <input id="groot-open-form-input" type="text" name="uri" value placeholder="URI to local or remote file">

		 Open

		<input id="groot-open-button" type="hidden" value="Open" onclick="openROOTFile()">

	{{- end}}

	<form id="groot-upload-form" enctype="multipart/form-data" action="/root-file-upload" method="post">

		 Upload

		<input id="groot-file-upload" type="file" name="groot-file"/>
		<input type="hidden" name="token" value="{{.Token}}"/>
		<input type="hidden" value="upload" />
	</form>

`
<|endoftext|>


<commit_before>// Copyright 2015 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumer

import (
	"github.com/trivago/gollum/log"
	"github.com/trivago/gollum/shared"
	"io"
	"os"
	"sync"
)

const (
	consoleBufferGrowSize = 256
)

// Console consumer plugin
// Configuration example
//
//   - "consumer.Console":
//     Enable: true
//
// This consumer does not define any options beside the standard ones.
type Console struct {
	shared.ConsumerBase
}

func init() {
	shared.RuntimeType.Register(Console{})
}

func (cons *Console) readFrom(stream io.Reader, threads *sync.WaitGroup) {
	buffer := shared.NewBufferedReader(consoleBufferGrowSize, 0, "\n", cons.PostMessageFromSlice)

	for {
		err := buffer.Read(stream)
		if err != nil {
			Log.Error.Print("Error reading stdin: ", err)
		}
	}
}

// Consume listens to stdin.
func (cons Console) Consume(threads *sync.WaitGroup) {
	go cons.readFrom(os.Stdin, threads)

	defer cons.MarkAsDone()
	cons.DefaultControlLoop(threads, nil)
}
<commit_msg>Fixed "ERROR: Console.go:50: Error reading stdin: EOF"<commit_after>// Copyright 2015 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumer

import (
	"github.com/trivago/gollum/log"
	"github.com/trivago/gollum/shared"
	"io"
	"os"
	"sync"
)

const (
	consoleBufferGrowSize = 256
)

// Console consumer plugin
// Configuration example
//
//   - "consumer.Console":
//     Enable: true
//
// This consumer does not define any options beside the standard ones.
type Console struct {
	shared.ConsumerBase
}

func init() {
	shared.RuntimeType.Register(Console{})
}

func (cons *Console) readFrom(stream io.Reader, threads *sync.WaitGroup) {
	buffer := shared.NewBufferedReader(consoleBufferGrowSize, 0, "\n", cons.PostMessageFromSlice)

	for {
		err := buffer.Read(stream)
		if err != nil && err != io.EOF {
			Log.Error.Print("Error reading stdin: ", err)
		}
	}
}

// Consume listens to stdin.
func (cons Console) Consume(threads *sync.WaitGroup) {
	go cons.readFrom(os.Stdin, threads)

	defer cons.MarkAsDone()
	cons.DefaultControlLoop(threads, nil)
}
<|endoftext|>


<commit_before>package gforms

import (
	"bytes"
	"net/http"
	"net/url"
	"reflect"
	"time"
	"bitbucket.org/spaaza/go-sdk"
)

type Form func(...*http.Request) *FormInstance

// cleaned data for all fields.
type CleanedData map[string]interface{}

// FormValidator is a function that will run only if all field cleaning and
// validation passes. It receives the CleanedData, providing an opportunity to
// validate multiple values together, and return an array of top level error strings.
type FormValidator func(cleanedData CleanedData) []string

// FormInstance made by Form.
type FormInstance struct {
	fieldInstances *FieldInterfaces
	Data           Data
	CleanedData    CleanedData
	ParseError     error
	errors         []string
	Validator      FormValidator
}

// Create a new form instance from `http.Request`.
func (f Form) FromRequest(r *http.Request) *FormInstance {
	return f(r)
}

// Create a new form instance from `url.Values`.
func (f Form) FromUrlValues(uv url.Values) *FormInstance {
	fi := f()
	fi.parseUrlValues(uv)
	return fi
}

// Get a `FieldInterface` for the given field name.
func (f *FormInstance) GetField(name string) (FieldInterface, bool) {
	v, ok := f.fieldInstances.nameMap[name]
	return v, ok
}

// Get all `FieldInstance` on `FormInstance`.
func (f *FormInstance) Fields() []FieldInterface {
	return f.fieldInstances.list
}

func (f *FormInstance) SetErrors(errors []string) {
	f.errors = errors
}

// Return field errors if any fields have error after calling `FormInstance#IsValid`.
// Also includes top level form errors.
func (f *FormInstance) Errors() Errors {
	errs := map[string][]string{}
	var err []string
	for _, field := range f.fieldInstances.list {
		name := field.GetModel().GetName()
		err = field.Errors()
		if len(err) > 0 {
			errs[name] = err
		}
	}
	formErr := f.FormErrors()
	if len(err) > 0 {
		errs["form"] = formErr
	}
	return errs
}

func (f *FormInstance) FormErrors() []string {
	return f.errors;
}

// Validation request data. If any fields have errors, this method returns false.
func (f *FormInstance) IsValid() bool {
	isValid := true
	f.CleanedData = CleanedData{}

	// validate fields
	for _, field := range f.fieldInstances.list {
		var err error
		name := field.GetModel().GetName()
		err = field.Clean(f.Data)
		if err != nil {
			field.SetErrors([]string{err.Error()})
			isValid = false
			continue
		}

		errs := field.Validate(f)
		if len(errs) > 0 {
			field.SetErrors(errs)
			isValid = false
			continue
		}

		if !field.GetV().IsNil {
			f.CleanedData[name] = field.GetV().Value
		}
	}

	// validate top level form
	if isValid && f.Validator != nil {
		formErrs := f.Validator(f.CleanedData)
		if len(formErrs) > 0 {
			isValid = false
			f.SetErrors(formErrs)
		}
	}

	return isValid
}

func (f *FormInstance) parseRequest(req *http.Request) error {
	data, err := bindRequest(req)
	if err != nil {
		return err
	}
	if data == nil {
		return nil
	}
	f.Data = *data
	return nil
}

func (f *FormInstance) parseUrlValues(uv url.Values) error {
	data, err := bindUrlValues(uv)
	if err != nil {
		return err
	}
	if data == nil {
		return nil
	}
	f.Data = *data
	return nil
}

// Get html of each FieldInstance.
func (f *FormInstance) Html() string {
	var html bytes.Buffer
	for _, field := range f.fieldInstances.list {
		html.WriteString(field.Html() + "\r\n")
	}
	return html.String()
}

// Define a new form with specified fields.
func DefineForm(fs *Fields) Form {
	return func(r ...*http.Request) *FormInstance {
		f := new(FormInstance)
		f.fieldInstances = newFieldInterfaces(fs)
		if len(r) > 0 {
			f.ParseError = f.parseRequest(r[0])
		}
		return f
	}
}

// maps cleanedData to specified model.
func (fi *FormInstance) MapTo(model interface{}) {
	if fi.CleanedData == nil {
		panic("MapTo method should be called after calling IsValid() method.")
	}
	if reflect.TypeOf(model).Kind() != reflect.Ptr {
		panic("Argument should be specified pointer type.")
	}
	mType := reflect.TypeOf(model).Elem()
	mValue := reflect.ValueOf(model).Elem()

	for i := 0; i < mValue.NumField(); i++ {
		typeField := mType.Field(i)
		tag := typeField.Tag.Get("gforms")
		if tag == "" {
			tag = typeField.Name
		} else if tag == "-" {
			continue
		}
		v, ok := fi.CleanedData[tag]
		if ok {
			valueField := mValue.Field(i)
			switch valueField.Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				value, ok := v.(int)
				if !ok {
					value = 0
				}
				valueField.SetInt(int64(value))
			case reflect.Float32, reflect.Float64:
				value, ok := v.(float64)
				if !ok {
					value = 0.0
				}
				valueField.SetFloat(value)
			case reflect.String:
				value, ok := v.(string)
				if !ok {
					va, ok := v.([]string)
					if ok && len(va) > 0 {
						value = va[0]
					} else {
						value = ""
					}
				}
				valueField.SetString(value)
			case reflect.Slice:
				value, ok := v.([]string)
				if !ok {
					value = []string{}
				}
				valueField.Set(reflect.ValueOf(value))
			case reflect.Bool:
				value, ok := v.(bool)
				if !ok {
					value = false
				}
				valueField.SetBool(value)
			case reflect.Struct:
				switch typeField.Type.String() {
				case "time.Time":
					value, ok := v.(time.Time)
					if !ok {
						value = time.Time{}
					}
					valueField.Set(reflect.ValueOf(value))
				case "spaazaSDK.CustomTime": // FIXME: This is bad!
				    value, ok := v.(time.Time)
				    if !ok {
				    	value = time.Time{}
				    }
				    valueField.Set(reflect.ValueOf(spaazaSDK.CustomTime{value}))
				}
			}
		}
	}
}
<commit_msg>Updated MapTo method to understand OptInProgramme and MailingList booleans from User model.<commit_after>package gforms

import (
	"bytes"
	"net/http"
	"net/url"
	"reflect"
	"time"
	"bitbucket.org/spaaza/go-sdk"
)

type Form func(...*http.Request) *FormInstance

// cleaned data for all fields.
type CleanedData map[string]interface{}

// FormValidator is a function that will run only if all field cleaning and
// validation passes. It receives the CleanedData, providing an opportunity to
// validate multiple values together, and return an array of top level error strings.
type FormValidator func(cleanedData CleanedData) []string

// FormInstance made by Form.
type FormInstance struct {
	fieldInstances *FieldInterfaces
	Data           Data
	CleanedData    CleanedData
	ParseError     error
	errors         []string
	Validator      FormValidator
}

// Create a new form instance from `http.Request`.
func (f Form) FromRequest(r *http.Request) *FormInstance {
	return f(r)
}

// Create a new form instance from `url.Values`.
func (f Form) FromUrlValues(uv url.Values) *FormInstance {
	fi := f()
	fi.parseUrlValues(uv)
	return fi
}

// Get a `FieldInterface` for the given field name.
func (f *FormInstance) GetField(name string) (FieldInterface, bool) {
	v, ok := f.fieldInstances.nameMap[name]
	return v, ok
}

// Get all `FieldInstance` on `FormInstance`.
func (f *FormInstance) Fields() []FieldInterface {
	return f.fieldInstances.list
}

func (f *FormInstance) SetErrors(errors []string) {
	f.errors = errors
}

// Return field errors if any fields have error after calling `FormInstance#IsValid`.
// Also includes top level form errors.
func (f *FormInstance) Errors() Errors {
	errs := map[string][]string{}
	var err []string
	for _, field := range f.fieldInstances.list {
		name := field.GetModel().GetName()
		err = field.Errors()
		if len(err) > 0 {
			errs[name] = err
		}
	}
	formErr := f.FormErrors()
	if len(err) > 0 {
		errs["form"] = formErr
	}
	return errs
}

func (f *FormInstance) FormErrors() []string {
	return f.errors;
}

// Validation request data. If any fields have errors, this method returns false.
func (f *FormInstance) IsValid() bool {
	isValid := true
	f.CleanedData = CleanedData{}

	// validate fields
	for _, field := range f.fieldInstances.list {
		var err error
		name := field.GetModel().GetName()
		err = field.Clean(f.Data)
		if err != nil {
			field.SetErrors([]string{err.Error()})
			isValid = false
			continue
		}

		errs := field.Validate(f)
		if len(errs) > 0 {
			field.SetErrors(errs)
			isValid = false
			continue
		}

		if !field.GetV().IsNil {
			f.CleanedData[name] = field.GetV().Value
		}
	}

	// validate top level form
	if isValid && f.Validator != nil {
		formErrs := f.Validator(f.CleanedData)
		if len(formErrs) > 0 {
			isValid = false
			f.SetErrors(formErrs)
		}
	}

	return isValid
}

func (f *FormInstance) parseRequest(req *http.Request) error {
	data, err := bindRequest(req)
	if err != nil {
		return err
	}
	if data == nil {
		return nil
	}
	f.Data = *data
	return nil
}

func (f *FormInstance) parseUrlValues(uv url.Values) error {
	data, err := bindUrlValues(uv)
	if err != nil {
		return err
	}
	if data == nil {
		return nil
	}
	f.Data = *data
	return nil
}

// Get html of each FieldInstance.
func (f *FormInstance) Html() string {
	var html bytes.Buffer
	for _, field := range f.fieldInstances.list {
		html.WriteString(field.Html() + "\r\n")
	}
	return html.String()
}

// Define a new form with specified fields.
func DefineForm(fs *Fields) Form {
	return func(r ...*http.Request) *FormInstance {
		f := new(FormInstance)
		f.fieldInstances = newFieldInterfaces(fs)
		if len(r) > 0 {
			f.ParseError = f.parseRequest(r[0])
		}
		return f
	}
}

// maps cleanedData to specified model.
func (fi *FormInstance) MapTo(model interface{}) {
	if fi.CleanedData == nil {
		panic("MapTo method should be called after calling IsValid() method.")
	}
	if reflect.TypeOf(model).Kind() != reflect.Ptr {
		panic("Argument should be specified pointer type.")
	}
	mType := reflect.TypeOf(model).Elem()
	mValue := reflect.ValueOf(model).Elem()

	for i := 0; i < mValue.NumField(); i++ {
		typeField := mType.Field(i)
		tag := typeField.Tag.Get("gforms")
		if tag == "" {
			tag = typeField.Name
		} else if tag == "-" {
			continue
		}
		v, ok := fi.CleanedData[tag]
		if ok {
			valueField := mValue.Field(i)
			switch valueField.Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				value, ok := v.(int)
				if !ok {
					value = 0
				}
				valueField.SetInt(int64(value))
			case reflect.Float32, reflect.Float64:
				value, ok := v.(float64)
				if !ok {
					value = 0.0
				}
				valueField.SetFloat(value)
			case reflect.String:
				value, ok := v.(string)
				if !ok {
					va, ok := v.([]string)
					if ok && len(va) > 0 {
						value = va[0]
					} else {
						value = ""
					}
				}
				valueField.SetString(value)
			case reflect.Slice:
				value, ok := v.([]string)
				if !ok {
					value = []string{}
				}
				valueField.Set(reflect.ValueOf(value))
			case reflect.Bool:
				value, ok := v.(bool)
				if !ok {
					value = false
				}
				valueField.SetBool(value)
			case reflect.Struct:
				switch typeField.Type.String() {
				case "time.Time":
					value, ok := v.(time.Time)
					if !ok {
						value = time.Time{}
					}
					valueField.Set(reflect.ValueOf(value))
				case "spaazaSDK.CustomTime": // FIXME: This is bad!
				    value, ok := v.(time.Time)
				    if !ok {
				    	value = time.Time{}
				    }
				    valueField.Set(reflect.ValueOf(spaazaSDK.CustomTime{value}))
				case "spaazaSDK.MailingList": // FIXME: This is bad bad!
					value, ok := v.(bool)
					if !ok {
						value = false
					}
					valueField.Set(reflect.ValueOf(spaazaSDK.MailingList{value}))
				case "spaazaSDK.OptInProgramme": // FIXME: This is bad bad bad!
					value, ok := v.(bool)
					if !ok {
						value = false
					}
					valueField.Set(reflect.ValueOf(spaazaSDK.OptInProgramme{value}))
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package websocket

import (
    "github.com/gorilla/websocket"
    "net/http"
    log "github.com/Sirupsen/logrus"
    parser ".././parse"
)

var upgrader = websocket.Upgrader {
    ReadBufferSize: 1024,
    WriteBufferSize: 1024,
}

func sendMessage(connection *websocket.Conn, message string) {
    connection.WriteMessage(1, []byte(message))
}

func Listen(port string) {
    http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        connection, err := upgrader.Upgrade(w, r, nil)
        sendMessage(connection, "Hello")

        if err != nil {
            log.Error(err)
            return
        }

        for {
            messageType, message, err := connection.ReadMessage()
            if messageType == -1 {
                return
            }
            if err != nil {
                log.Error(err)
                return
            }

            msg, err := parser.Parse([]byte(message))
            if err != nil {
                log.Error(err)
                return
            }

            if msg != nil {
                log.Info("Got: ", msg)
                log.Info("Got: ", msg.Type)
                log.Info("Got: ", msg.Method)
                log.Info("Got: ", msg.Method["salad"])
                log.Info("Got: ", msg.Method["potato"])
            }
        }
    })

    http.ListenAndServe(port, nil)
}
<commit_msg>Remove debug code<commit_after>package websocket

import (
    "github.com/gorilla/websocket"
    "net/http"
    log "github.com/Sirupsen/logrus"
    parser ".././parse"
)

var upgrader = websocket.Upgrader {
    ReadBufferSize: 1024,
    WriteBufferSize: 1024,
}

func sendMessage(connection *websocket.Conn, message string) {
    connection.WriteMessage(1, []byte(message))
}

func Listen(port string) {
    http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        connection, err := upgrader.Upgrade(w, r, nil)
        sendMessage(connection, "Hello")

        if err != nil {
            log.Error(err)
            return
        }

        for {
            messageType, message, err := connection.ReadMessage()
            if messageType == -1 {
                return
            }
            if err != nil {
                log.Error(err)
                return
            }

            msg, err := parser.Parse([]byte(message))
            if err != nil {
                log.Error(err)
                return
            }

            if msg != nil {
                log.Info("Got: ", msg)
            }
        }
    })

    http.ListenAndServe(port, nil)
}
<|endoftext|>


<commit_before>package websocket

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/http2"

	"github.com/cactusbot/sepal/client"
	"github.com/cactusbot/sepal/client/ratelimit"
	"github.com/cactusbot/sepal/database"
	parser "github.com/cactusbot/sepal/parse"
	"github.com/cactusbot/sepal/util"
	"github.com/gorilla/websocket"
)

var (
	upgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
	}

	log = util.GetLogger()

	Close = make(chan bool)
)

type commandPacket struct {
	Scope    string `json:"scope"`
	Command  string `json:"command"`
	Response string `json:"response"`
	ID       string `json:"id"`
	Channel  string `json:"channel"`
}

type quotePacket struct {
	Scope    string `json:"scope"`
	ID       int    `json:"id"`
	Response string `json:"response"`
	Channel  string `json:"channel"`
}

func sendMessage(connection *websocket.Conn, message string) {
	connection.WriteMessage(1, []byte(message))
}

// Dispatch - Dispatch a message
func Dispatch() {
	for {
		select {
		case command := <-database.CommandChannel:
			var packet commandPacket

			if command.NewVal == nil && command.OldVal != nil {
				packet = commandPacket{
					Scope:    "command:remove",
					Command:  command.OldVal.Command,
					Response: command.OldVal.Response,
					ID:       command.OldVal.ID,
					Channel:  command.OldVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope("command:remove", packet.Channel, string(data))
			} else {
				packet = commandPacket{
					Scope:    "command:create",
					Command:  command.NewVal.Command,
					Response: command.NewVal.Response,
					ID:       command.NewVal.ID,
					Channel:  command.NewVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope("command:create", packet.Channel, string(data))
			}
		case quote := <-database.QuoteChannel:
			var packet quotePacket

			if quote.OldVal != nil && quote.NewVal == nil {
				packet = quotePacket{
					Scope:    "quote:remove",
					ID:       quote.OldVal.ID,
					Response: quote.OldVal.Quote,
					Channel:  quote.OldVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope("quote:remove", packet.Channel, string(data))
			} else {
				packet = quotePacket{
					Scope:    "quote:create",
					ID:       quote.NewVal.ID,
					Response: quote.NewVal.Quote,
					Channel:  quote.NewVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope("quote:create", packet.Channel, string(data))
			}
		}
	}
}

// Listen listen for websocket connections
func Listen(port string) {
	var server http.Server
	http2.VerboseLogs = true
	server.Addr = port

	http2.ConfigureServer(&server, nil)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		connection, err := upgrader.Upgrade(w, r, nil)
		ip := connection.LocalAddr().String()

		if err != nil {
			return
		}

		if !ratelimit.CanConnect(ip) {
			go func(conn *websocket.Conn) {
				err = conn.Close()
				if err != nil {
					log.Error(err)
				}
			}(connection)
		}

		log.Info("Connection from IP: ", ip)
		log.Debug("Remaining Limit: ", ratelimit.GetLimit(ip))

		packet := map[string]string{
			"type": "aloha",
		}
		packetMsg, _ := json.Marshal(packet)

		sendMessage(connection, string(packetMsg))

		for {
			_, message, err := connection.ReadMessage()

			if err != nil {
				log.Info("Client with IP ", ip, " disconnected.")
				client.Clients[ip] = client.Client{}
			}

			if err != nil {
				log.Error(err)
			}

			go func() {
				msg, err := parser.Parse([]byte(message))
				if err != nil {
					log.Error(err)
				}

				if msg != nil {
					var currentClient client.Client

					log.Debug("Got a packet: ", msg)
					if msg.Type == "auth" {
						currentClient = client.Client{
							Scopes:     msg.Scopes,
							IP:         ip,
							Connection: connection,
							Channel:    msg.Channel,
						}
						client.AddClient(&currentClient)

						log.Info("Client with the ip of: ",
							currentClient.IP, " subscribed to scopes: ",
							currentClient.Scopes, " of the channel: ",
							currentClient.Channel)
					} else {
						packet = map[string]string{
							"type":  "error",
							"error": "Invalid packet type: " + msg.Type,
						}
						packetMsg, _ := json.Marshal(packet)

						sendMessage(connection, string(packetMsg))
					}

				}
			}()
		}
	})

	log.Fatal(server.ListenAndServe())
	// log.Fatal(server.ListenAndServeTLS("cert.pem", "key.pem"))
}
<commit_msg>Fix sending scopes<commit_after>package websocket

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/http2"

	"github.com/cactusbot/sepal/client"
	"github.com/cactusbot/sepal/client/ratelimit"
	"github.com/cactusbot/sepal/database"
	parser "github.com/cactusbot/sepal/parse"
	"github.com/cactusbot/sepal/util"
	"github.com/gorilla/websocket"
)

var (
	upgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
	}

	log = util.GetLogger()

	Close = make(chan bool)
)

type commandPacket struct {
	Scope    string `json:"scope"`
	Command  string `json:"command"`
	Response string `json:"response"`
	ID       string `json:"id"`
	Channel  string `json:"channel"`
}

type quotePacket struct {
	Scope    string `json:"scope"`
	ID       int    `json:"id"`
	Response string `json:"response"`
	Channel  string `json:"channel"`
}

func sendMessage(connection *websocket.Conn, message string) {
	connection.WriteMessage(1, []byte(message))
}

// Dispatch - Dispatch a message
func Dispatch() {
	for {
		select {
		case command := <-database.CommandChannel:
			var packet commandPacket

			if command.NewVal == nil && command.OldVal != nil {
				packet = commandPacket{
					Scope:    "command:remove",
					Command:  command.OldVal.Command,
					Response: command.OldVal.Response,
					ID:       command.OldVal.ID,
					Channel:  command.OldVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope("command:remove", packet.Channel, string(data))
			} else {
				packet = commandPacket{
					Scope:    "command:create",
					Command:  command.NewVal.Command,
					Response: command.NewVal.Response,
					ID:       command.NewVal.ID,
					Channel:  command.NewVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope("command:create", packet.Channel, string(data))
			}
		case quote := <-database.QuoteChannel:
			var packet quotePacket

			if quote.OldVal != nil && quote.NewVal == nil {
				packet = quotePacket{
					Scope:    "quote:remove",
					ID:       quote.OldVal.ID,
					Response: quote.OldVal.Quote,
					Channel:  quote.OldVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope(packet.Scope, packet.Channel, string(data))
			} else {
				packet = quotePacket{
					Scope:    "quote:create",
					ID:       quote.NewVal.ID,
					Response: quote.NewVal.Quote,
					Channel:  quote.NewVal.Channel,
				}

				data, _ := json.Marshal(packet)
				client.BroadcastToScope(packet.Scope, packet.Channel, string(data))
			}
		}
	}
}

// Listen listen for websocket connections
func Listen(port string) {
	var server http.Server
	http2.VerboseLogs = true
	server.Addr = port

	http2.ConfigureServer(&server, nil)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		connection, err := upgrader.Upgrade(w, r, nil)
		ip := connection.LocalAddr().String()

		if err != nil {
			return
		}

		if !ratelimit.CanConnect(ip) {
			go func(conn *websocket.Conn) {
				err = conn.Close()
				if err != nil {
					log.Error(err)
				}
			}(connection)
		}

		log.Info("Connection from IP: ", ip)
		log.Debug("Remaining Limit: ", ratelimit.GetLimit(ip))

		packet := map[string]string{
			"type": "aloha",
		}
		packetMsg, _ := json.Marshal(packet)

		sendMessage(connection, string(packetMsg))

		for {
			_, message, err := connection.ReadMessage()

			if err != nil {
				log.Info("Client with IP ", ip, " disconnected.")
				client.Clients[ip] = client.Client{}
			}

			if err != nil {
				log.Error(err)
			}

			go func() {
				msg, err := parser.Parse([]byte(message))
				if err != nil {
					log.Error(err)
				}

				if msg != nil {
					var currentClient client.Client

					log.Debug("Got a packet: ", msg)
					if msg.Type == "auth" {
						currentClient = client.Client{
							Scopes:     msg.Scopes,
							IP:         ip,
							Connection: connection,
							Channel:    msg.Channel,
						}
						client.AddClient(&currentClient)

						log.Info("Client with the ip of: ",
							currentClient.IP, " subscribed to scopes: ",
							currentClient.Scopes, " of the channel: ",
							currentClient.Channel)
					} else {
						packet = map[string]string{
							"type":  "error",
							"error": "Invalid packet type: " + msg.Type,
						}
						packetMsg, _ := json.Marshal(packet)

						sendMessage(connection, string(packetMsg))
					}

				}
			}()
		}
	})

	log.Fatal(server.ListenAndServe())
	// log.Fatal(server.ListenAndServeTLS("cert.pem", "key.pem"))
}
<|endoftext|>


<commit_before>package engine

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestRegister(t *testing.T) {
	if err := Register("dummy1", nil); err != nil {
		t.Fatal(err)
	}

	if err := Register("dummy1", nil); err == nil {
		t.Fatalf("Expecting error, got none")
	}

	eng := newTestEngine(t)

	//Should fail because global handlers are copied
	//at the engine creation
	if err := eng.Register("dummy1", nil); err == nil {
		t.Fatalf("Expecting error, got none")
	}

	if err := eng.Register("dummy2", nil); err != nil {
		t.Fatal(err)
	}

	if err := eng.Register("dummy2", nil); err == nil {
		t.Fatalf("Expecting error, got none")
	}
}

func TestJob(t *testing.T) {
	eng := newTestEngine(t)
	job1 := eng.Job("dummy1", "--level=awesome")

	if job1.handler != nil {
		t.Fatalf("job1.handler should be empty")
	}

	h := func(j *Job) Status {
		j.Printf("%s\n", j.Name)
		return 42
	}

	eng.Register("dummy2", h)
	job2 := eng.Job("dummy2", "--level=awesome")

	if job2.handler == nil {
		t.Fatalf("job2.handler shouldn't be nil")
	}

	if job2.handler(job2) != 42 {
		t.Fatalf("handler dummy2 was not found in job2")
	}
}

func TestEngineRoot(t *testing.T) {
	tmp, err := ioutil.TempDir("", "docker-test-TestEngineCreateDir")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)
	dir := path.Join(tmp, "dir")
	eng, err := New(dir)
	if err != nil {
		t.Fatal(err)
	}
	if st, err := os.Stat(dir); err != nil {
		t.Fatal(err)
	} else if !st.IsDir() {
		t.Fatalf("engine.New() created something other than a directory at %s", dir)
	}
	if r := eng.Root(); r != dir {
		t.Fatalf("Expected: %v\nReceived: %v", dir, r)
	}
}

func TestEngineString(t *testing.T) {
	eng1 := newTestEngine(t)
	defer os.RemoveAll(eng1.Root())
	eng2 := newTestEngine(t)
	defer os.RemoveAll(eng2.Root())
	s1 := eng1.String()
	s2 := eng2.String()
	if eng1 == eng2 {
		t.Fatalf("Different engines should have different names (%v == %v)", s1, s2)
	}
}

func TestEngineLogf(t *testing.T) {
	eng := newTestEngine(t)
	defer os.RemoveAll(eng.Root())
	input := "Test log line"
	if n, err := eng.Logf("%s\n", input); err != nil {
		t.Fatal(err)
	} else if n < len(input) {
		t.Fatalf("Test: Logf() should print at least as much as the input\ninput=%d\nprinted=%d", len(input), n)
	}
}
<commit_msg>engine: fix test on mac<commit_after>package engine

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"testing"
)

func TestRegister(t *testing.T) {
	if err := Register("dummy1", nil); err != nil {
		t.Fatal(err)
	}

	if err := Register("dummy1", nil); err == nil {
		t.Fatalf("Expecting error, got none")
	}

	eng := newTestEngine(t)

	//Should fail because global handlers are copied
	//at the engine creation
	if err := eng.Register("dummy1", nil); err == nil {
		t.Fatalf("Expecting error, got none")
	}

	if err := eng.Register("dummy2", nil); err != nil {
		t.Fatal(err)
	}

	if err := eng.Register("dummy2", nil); err == nil {
		t.Fatalf("Expecting error, got none")
	}
}

func TestJob(t *testing.T) {
	eng := newTestEngine(t)
	job1 := eng.Job("dummy1", "--level=awesome")

	if job1.handler != nil {
		t.Fatalf("job1.handler should be empty")
	}

	h := func(j *Job) Status {
		j.Printf("%s\n", j.Name)
		return 42
	}

	eng.Register("dummy2", h)
	job2 := eng.Job("dummy2", "--level=awesome")

	if job2.handler == nil {
		t.Fatalf("job2.handler shouldn't be nil")
	}

	if job2.handler(job2) != 42 {
		t.Fatalf("handler dummy2 was not found in job2")
	}
}

func TestEngineRoot(t *testing.T) {
	tmp, err := ioutil.TempDir("", "docker-test-TestEngineCreateDir")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)
	dir := path.Join(tmp, "dir")
	eng, err := New(dir)
	if err != nil {
		t.Fatal(err)
	}
	if st, err := os.Stat(dir); err != nil {
		t.Fatal(err)
	} else if !st.IsDir() {
		t.Fatalf("engine.New() created something other than a directory at %s", dir)
	}
	r := eng.Root()
	r, _ = filepath.EvalSymlinks(r)
	dir, _ = filepath.EvalSymlinks(dir)
	if r != dir {
		t.Fatalf("Expected: %v\nReceived: %v", dir, r)
	}
}

func TestEngineString(t *testing.T) {
	eng1 := newTestEngine(t)
	defer os.RemoveAll(eng1.Root())
	eng2 := newTestEngine(t)
	defer os.RemoveAll(eng2.Root())
	s1 := eng1.String()
	s2 := eng2.String()
	if eng1 == eng2 {
		t.Fatalf("Different engines should have different names (%v == %v)", s1, s2)
	}
}

func TestEngineLogf(t *testing.T) {
	eng := newTestEngine(t)
	defer os.RemoveAll(eng.Root())
	input := "Test log line"
	if n, err := eng.Logf("%s\n", input); err != nil {
		t.Fatal(err)
	} else if n < len(input) {
		t.Fatalf("Test: Logf() should print at least as much as the input\ninput=%d\nprinted=%d", len(input), n)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"net/http"
	httppprof "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/viper"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util/httpdown"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	v VolumeServerOptions
)

type VolumeServerOptions struct {
	port                  *int
	publicPort            *int
	folders               []string
	folderMaxLimits       []int
	ip                    *string
	publicUrl             *string
	bindIp                *string
	masters               *string
	idleConnectionTimeout *int
	dataCenter            *string
	rack                  *string
	whiteList             []string
	indexType             *string
	fixJpgOrientation     *bool
	readRedirect          *bool
	cpuProfile            *string
	memProfile            *string
	compactionMBPerSecond *int
	fileSizeLimitMB       *int
	minFreeSpacePercent   []float32
	pprof                 *bool
	// pulseSeconds          *int
}

func init() {
	cmdVolume.Run = runVolume // break init cycle
	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]... If set to zero on non-windows OS, the limit will be auto configured.")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent ", "0", "minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly")
)

func runVolume(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)

	runtime.GOMAXPROCS(runtime.NumCPU())

	// If --pprof is set we assume the caller wants to be able to collect
	// cpu and memory profiles via go tool pprof
	if !*v.pprof {
		grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
	}

	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)

	return true
}

func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {

	// Set multiple folders and each folder's max volume count limit'
	v.folders = strings.Split(volumeFolders, ",")
	maxCountStrings := strings.Split(maxVolumeCounts, ",")
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			v.folderMaxLimits = append(v.folderMaxLimits, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(v.folders) != len(v.folderMaxLimits) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
	}
	minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
	for _, freeString := range minFreeSpacePercentStrings {

		if value, e := strconv.ParseFloat(freeString, 32); e == nil {
			v.minFreeSpacePercent = append(v.minFreeSpacePercent, float32(value))
		} else {
			glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
		}
	}

	for _, folder := range v.folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	// security related white list configuration
	if volumeWhiteListOption != "" {
		v.whiteList = strings.Split(volumeWhiteListOption, ",")
	}

	if *v.ip == "" {
		*v.ip = util.DetectedHostAddress()
		glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
	}

	if *v.publicPort == 0 {
		*v.publicPort = *v.port
	}
	if *v.publicUrl == "" {
		*v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
	}

	volumeMux := http.NewServeMux()
	publicVolumeMux := volumeMux
	if v.isSeparatedPublicPort() {
		publicVolumeMux = http.NewServeMux()
	}

	if *v.pprof {
		volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
		volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
		volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
		volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
		volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
	}

	volumeNeedleMapKind := storage.NeedleMapInMemory
	switch *v.indexType {
	case "leveldb":
		volumeNeedleMapKind = storage.NeedleMapLevelDb
	case "leveldbMedium":
		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
	case "leveldbLarge":
		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
	}

	masters := *v.masters

	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
		*v.ip, *v.port, *v.publicUrl,
		v.folders, v.folderMaxLimits, v.minFreeSpacePercent,
		volumeNeedleMapKind,
		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
		v.whiteList,
		*v.fixJpgOrientation, *v.readRedirect,
		*v.compactionMBPerSecond,
		*v.fileSizeLimitMB,
	)

	// starting grpc server
	grpcS := v.startGrpcService(volumeServer)

	// starting public http server
	var publicHttpDown httpdown.Server
	if v.isSeparatedPublicPort() {
		publicHttpDown = v.startPublicHttpService(publicVolumeMux)
		if nil == publicHttpDown {
			glog.Fatalf("start public http service failed")
		}
	}

	// starting the cluster http server
	clusterHttpServer := v.startClusterHttpService(volumeMux)

	stopChain := make(chan struct{})
	grace.OnInterrupt(func() {
		fmt.Println("volume server has be killed")
		var startTime time.Time

		// firstly, stop the public http service to prevent from receiving new user request
		if nil != publicHttpDown {
			startTime = time.Now()
			if err := publicHttpDown.Stop(); err != nil {
				glog.Warningf("stop the public http server failed, %v", err)
			}
			delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
			glog.V(0).Infof("stop public http server, elapsed %dms", delta)
		}

		startTime = time.Now()
		if err := clusterHttpServer.Stop(); err != nil {
			glog.Warningf("stop the cluster http server failed, %v", err)
		}
		delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("graceful stop cluster http server, elapsed [%d]", delta)

		startTime = time.Now()
		grpcS.GracefulStop()
		delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("graceful stop gRPC, elapsed [%d]", delta)

		startTime = time.Now()
		volumeServer.Shutdown()
		delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("stop volume server, elapsed [%d]", delta)

		pprof.StopCPUProfile()

		close(stopChain) // notify exit
	})

	select {
	case <-stopChain:
	}
	glog.Warningf("the volume server exit.")
}

// check whether configure the public port
func (v VolumeServerOptions) isSeparatedPublicPort() bool {
	return *v.publicPort != *v.port
}

func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
	grpcPort := *v.port + 10000
	grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
	volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
	reflection.Register(grpcS)
	go func() {
		if err := grpcS.Serve(grpcL); err != nil {
			glog.Fatalf("start gRPC service failed, %s", err)
		}
	}()
	return grpcS
}

func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
	publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
	publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
	publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
	go func() {
		if err := publicHttpDown.Wait(); err != nil {
			glog.Errorf("public http down wait failed, %v", err)
		}
	}()

	return publicHttpDown
}

func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
	var (
		certFile, keyFile string
	)
	if viper.GetString("https.volume.key") != "" {
		certFile = viper.GetString("https.volume.cert")
		keyFile = viper.GetString("https.volume.key")
	}

	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	httpDown := httpdown.HTTP{
		KillTimeout: 5 * time.Minute,
		StopTimeout: 5 * time.Minute,
		CertFile:    certFile,
		KeyFile:     keyFile}
	clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
	go func() {
		if e := clusterHttpServer.Wait(); e != nil {
			glog.Fatalf("Volume server fail to serve: %v", e)
		}
	}()
	return clusterHttpServer
}
<commit_msg>volume: fix typo in minFreeSpacePercent option<commit_after>package command

import (
	"fmt"
	"net/http"
	httppprof "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/viper"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util/httpdown"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	v VolumeServerOptions
)

type VolumeServerOptions struct {
	port                  *int
	publicPort            *int
	folders               []string
	folderMaxLimits       []int
	ip                    *string
	publicUrl             *string
	bindIp                *string
	masters               *string
	idleConnectionTimeout *int
	dataCenter            *string
	rack                  *string
	whiteList             []string
	indexType             *string
	fixJpgOrientation     *bool
	readRedirect          *bool
	cpuProfile            *string
	memProfile            *string
	compactionMBPerSecond *int
	fileSizeLimitMB       *int
	minFreeSpacePercent   []float32
	pprof                 *bool
	// pulseSeconds          *int
}

func init() {
	cmdVolume.Run = runVolume // break init cycle
	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]... If set to zero on non-windows OS, the limit will be auto configured.")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "0", "minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly")
)

func runVolume(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)

	runtime.GOMAXPROCS(runtime.NumCPU())

	// If --pprof is set we assume the caller wants to be able to collect
	// cpu and memory profiles via go tool pprof
	if !*v.pprof {
		grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
	}

	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)

	return true
}

func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {

	// Set multiple folders and each folder's max volume count limit'
	v.folders = strings.Split(volumeFolders, ",")
	maxCountStrings := strings.Split(maxVolumeCounts, ",")
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			v.folderMaxLimits = append(v.folderMaxLimits, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(v.folders) != len(v.folderMaxLimits) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
	}
	minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
	for _, freeString := range minFreeSpacePercentStrings {

		if value, e := strconv.ParseFloat(freeString, 32); e == nil {
			v.minFreeSpacePercent = append(v.minFreeSpacePercent, float32(value))
		} else {
			glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
		}
	}

	for _, folder := range v.folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	// security related white list configuration
	if volumeWhiteListOption != "" {
		v.whiteList = strings.Split(volumeWhiteListOption, ",")
	}

	if *v.ip == "" {
		*v.ip = util.DetectedHostAddress()
		glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
	}

	if *v.publicPort == 0 {
		*v.publicPort = *v.port
	}
	if *v.publicUrl == "" {
		*v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
	}

	volumeMux := http.NewServeMux()
	publicVolumeMux := volumeMux
	if v.isSeparatedPublicPort() {
		publicVolumeMux = http.NewServeMux()
	}

	if *v.pprof {
		volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
		volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
		volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
		volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
		volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
	}

	volumeNeedleMapKind := storage.NeedleMapInMemory
	switch *v.indexType {
	case "leveldb":
		volumeNeedleMapKind = storage.NeedleMapLevelDb
	case "leveldbMedium":
		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
	case "leveldbLarge":
		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
	}

	masters := *v.masters

	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
		*v.ip, *v.port, *v.publicUrl,
		v.folders, v.folderMaxLimits, v.minFreeSpacePercent,
		volumeNeedleMapKind,
		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
		v.whiteList,
		*v.fixJpgOrientation, *v.readRedirect,
		*v.compactionMBPerSecond,
		*v.fileSizeLimitMB,
	)

	// starting grpc server
	grpcS := v.startGrpcService(volumeServer)

	// starting public http server
	var publicHttpDown httpdown.Server
	if v.isSeparatedPublicPort() {
		publicHttpDown = v.startPublicHttpService(publicVolumeMux)
		if nil == publicHttpDown {
			glog.Fatalf("start public http service failed")
		}
	}

	// starting the cluster http server
	clusterHttpServer := v.startClusterHttpService(volumeMux)

	stopChain := make(chan struct{})
	grace.OnInterrupt(func() {
		fmt.Println("volume server has be killed")
		var startTime time.Time

		// firstly, stop the public http service to prevent from receiving new user request
		if nil != publicHttpDown {
			startTime = time.Now()
			if err := publicHttpDown.Stop(); err != nil {
				glog.Warningf("stop the public http server failed, %v", err)
			}
			delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
			glog.V(0).Infof("stop public http server, elapsed %dms", delta)
		}

		startTime = time.Now()
		if err := clusterHttpServer.Stop(); err != nil {
			glog.Warningf("stop the cluster http server failed, %v", err)
		}
		delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("graceful stop cluster http server, elapsed [%d]", delta)

		startTime = time.Now()
		grpcS.GracefulStop()
		delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("graceful stop gRPC, elapsed [%d]", delta)

		startTime = time.Now()
		volumeServer.Shutdown()
		delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("stop volume server, elapsed [%d]", delta)

		pprof.StopCPUProfile()

		close(stopChain) // notify exit
	})

	select {
	case <-stopChain:
	}
	glog.Warningf("the volume server exit.")
}

// check whether configure the public port
func (v VolumeServerOptions) isSeparatedPublicPort() bool {
	return *v.publicPort != *v.port
}

func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
	grpcPort := *v.port + 10000
	grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
	volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
	reflection.Register(grpcS)
	go func() {
		if err := grpcS.Serve(grpcL); err != nil {
			glog.Fatalf("start gRPC service failed, %s", err)
		}
	}()
	return grpcS
}

func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
	publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
	publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
	publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
	go func() {
		if err := publicHttpDown.Wait(); err != nil {
			glog.Errorf("public http down wait failed, %v", err)
		}
	}()

	return publicHttpDown
}

func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
	var (
		certFile, keyFile string
	)
	if viper.GetString("https.volume.key") != "" {
		certFile = viper.GetString("https.volume.cert")
		keyFile = viper.GetString("https.volume.key")
	}

	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	httpDown := httpdown.HTTP{
		KillTimeout: 5 * time.Minute,
		StopTimeout: 5 * time.Minute,
		CertFile:    certFile,
		KeyFile:     keyFile}
	clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
	go func() {
		if e := clusterHttpServer.Wait(); e != nil {
			glog.Fatalf("Volume server fail to serve: %v", e)
		}
	}()
	return clusterHttpServer
}
<|endoftext|>


<commit_before>package goridge

import (
	"unsafe"
)

const FRAME_OPTIONS_MAX_SIZE = 40 //nolint:golint
const WORD = 4                    //nolint:golint

// Frame defines new user level package format.
type Frame struct {
	// Payload, max length 4.2GB.
	payload []byte

	// Header
	header []byte
}

// ReadHeader reads only header, without payload
func ReadHeader(data []byte) *Frame {
	if lookupTable[1] == 0 {
		panic("should initialize lookup table")
	}
	//_ = data[0]
	//opt := data[0] & 0x0F
	//// if more than 2, that we have options
	//if opt > 2 {
	//	return &Frame{
	//		header:  data[:opt*WORD],
	//		payload: nil,
	//	}
	//}

	// no options
	return &Frame{
		header:  data[:8],
		payload: nil,
	}
}

func ReadFrame(data []byte) *Frame {
	if lookupTable[1] == 0 {
		panic("should initialize lookup table")
	}
	_ = data[0]
	opt := data[0] & 0x0F
	// if more than 2, that we have options
	if opt > 2 {
		return &Frame{
			header:  data[:opt*WORD],
			payload: data[opt*WORD:],
		}
	}

	// no options
	return &Frame{
		header:  data[:8],
		payload: data[8:],
	}
}

func NewFrame() *Frame {
	if lookupTable[1] == 0 {
		panic("should initialize lookup table")
	}
	f := &Frame{
		header:  make([]byte, 8),
		payload: make([]byte, 0, 100),
	}
	// set default header len (5)
	f.defaultHL()
	return f
}

// MergeHeader merge header from other frame with original payload
func (f *Frame) MergeHeader(frame *Frame) {
	f.header = frame.header
}

// To read version, we should return our 4 upper bits to their original place
// 1111_0000 -> 0000_1111 (15)
func (f *Frame) ReadVersion() byte {
	_ = f.header[0]
	return f.header[0] >> 4
}

// To write version, we should made the following:
// 1. For example we have version 15 it's 0000_1111 (1 byte)
// 2. We should shift 4 lower bits to upper and write that to the 0th byte
// 3. The 0th byte should become 1111_0000, but it's not 240, it's only 15, because version only 4 bits len
func (f *Frame) WriteVersion(version Version) {
	_ = f.header[0]
	if version > 15 {
		panic("version is only 4 bits")
	}
	f.header[0] = byte(version)<<4 | f.header[0]
}

// The lower 4 bits of the 0th octet occupies our header len data.
// We should erase upper 4 bits, which contain information about Version
// To erase, we applying bitwise AND to the upper 4 bits and returning result
func (f *Frame) readHL() byte {
	_ = f.header[0]
	// 0101_1111         0000_1111
	return f.header[0] & 0x0F
}

// Writing HL is very simple. Since we are using lower 4 bits
// we can easily apply bitwise OR and set lower 4 bits to needed hl value
func (f *Frame) writeHl(hl byte) {
	_ = f.header[0]
	f.header[0] = f.header[0] | hl
}

func (f *Frame) incrementHL() {
	_ = f.header[0]
	hl := f.readHL()
	if hl > 15 {
		panic("header len should be less than 15")
	}
	f.header[0] = f.header[0] | hl + 1
}
func (f *Frame) defaultHL() {
	_ = f.header[0]
	f.writeHl(2)
}

// Flags is full 1st byte
func (f *Frame) ReadFlags() uint8 {
	_ = f.header[1]
	return f.header[1]
}

func (f *Frame) WriteFlags(flags ...FrameFlag) {
	_ = f.header[1]
	for i := 0; i < len(flags); i++ {
		f.header[1] = f.header[1] | byte(flags[i])
	}
}

// Options slice len should not be more than 10 (40 bytes)
func (f *Frame) WriteOption(options ...uint32) {
	hl := f.readHL()
	// check before writing. we can't handle more than 15*4 bytes of HL (2 for header and 12 for options)
	if hl == 15 {
		panic("header len could not be more than 15")
	}
	if len(options) > 10 {
		panic("header options limited by 40 bytes")
	}

	tmp := make([]byte, 0, FRAME_OPTIONS_MAX_SIZE)
	for i := 0; i < len(options); i++ {
		tmp = append(tmp, byte(options[i]))
		tmp = append(tmp, byte(options[i]>>8))
		tmp = append(tmp, byte(options[i]>>16))
		tmp = append(tmp, byte(options[i]>>24))
		f.incrementHL() // increment header len by 32 bit
	}

	f.header = append(f.header, tmp...)
}

// AppendOptions appends options to the header
func (f *Frame) AppendOptions(opts []byte) {
	f.header = append(f.header, opts...)
}

// f.readHL() - 2 needed to know actual options size
// we know, that 2 WORDS is minimal header len
// extra WORDS will add extra 32bits to the options (4 bytes)
//
func (f *Frame) ReadOptions() []uint32 {
	// we can read options, if there are no options
	if f.readHL() <= 2 {
		return nil
	}
	// Get the options len
	optionLen := f.readHL() - 2 // 2 is the default
	// slice in place
	options := make([]uint32, 0, optionLen)

	// Options starting from 8-th byte
	// we should scan with 4 byte window (32bit, WORD)
	for i := byte(0); i < optionLen*WORD; i += WORD {
		// for example
		// 8  12  16
		// 9  13  17
		// 10 14  18
		// 11 15  19
		// For this data, HL will be 3, optionLen will be 12 (3*4) bytes
		options = append(options, uint32(f.header[8+i])|uint32(f.header[8+i+1])<<8|uint32(f.header[8+i+2])<<16|uint32(f.header[8+i+3])<<24)
	}
	return options
}

// LE format used to write Payload
// Using 4 bytes (2,3,4,5 bytes in the header)
func (f *Frame) ReadPayloadLen() uint32 {
	// 2,3,4,5
	_ = f.header[5]
	return uint32(f.header[2]) | uint32(f.header[3])<<8 | uint32(f.header[4])<<16 | uint32(f.header[5])<<24
}

// LE format used to write Payload
// Using 4 bytes (2,3,4,5 bytes in the header)
func (f *Frame) WritePayloadLen(len uint32) {
	_ = f.header[5]
	f.header[2] = byte(len)
	f.header[3] = byte(len >> 8)
	f.header[4] = byte(len >> 16)
	f.header[5] = byte(len >> 24)
}

// Calculating CRC and writing it to the 6th byte (7th reserved)
func (f *Frame) WriteCRC() {
	_ = f.header[7]
	crc := byte(0)
	hl := f.readHL()
	// write CRC with options
	if f.readHL() > 2 {
		for i := byte(0); i < hl*WORD; i++ {
			data := f.header[i] ^ crc
			crc = lookupTable[data]
		}
		f.header[6] = crc
		return
	}

	for i := 0; i < 6; i++ {
		data := f.header[i] ^ crc
		crc = lookupTable[data]
	}

	f.header[6] = crc
}

// Reading info from 6th byte and verifying it with calculated in-place. Should be equal.
// If not - drop the frame as incorrect.
func (f *Frame) VerifyCRC() bool {
	_ = f.header[7]
	crc := byte(0)
	hl := f.readHL()
	if hl > 2 {
		for i := byte(0); i < hl*WORD; i++ {
			// to verify, we are skipping the CRC field itself
			if i == 6 {
				data := 0 ^ crc
				crc = lookupTable[data]
				continue
			}
			data := f.header[i] ^ crc
			crc = lookupTable[data]
		}
		return crc == f.header[6]
	}

	for i := 0; i < 6; i++ {
		data := f.header[i] ^ crc
		crc = lookupTable[data]
	}
	return crc == f.header[6]
}

// Bytes returns header with payload
func (f *Frame) Bytes() []byte {
	buf := make([]byte, 0, len(f.header)+len(f.payload))
	buf = append(buf, f.header...)
	buf = append(buf, f.payload...)
	return buf
}

// Header returns frame header
func (f *Frame) Header() []byte {
	return f.header
}

// Payload returns frame payload without header
func (f *Frame) Payload() []byte {
	// start from the 1st (staring from 0) byte
	return f.payload
}

//
func (f *Frame) WritePayload(data []byte) {
	f.payload = make([]byte, len(data))
	copy(f.payload, data)
}

//go:nosplit
//go:nocheckptr
func noescape(p unsafe.Pointer) unsafe.Pointer {
	x := uintptr(p)
	return unsafe.Pointer(x ^ 0) //nolint:staticcheck
}

// After reset you should write all data from the start
func (f *Frame) Reset() {
	f.header = make([]byte, 0, 8)
	f.payload = make([]byte, 0, 100)
}
<commit_msg>Add options guard to writeoptions method<commit_after>package goridge

import (
	"unsafe"
)

const FRAME_OPTIONS_MAX_SIZE = 40 //nolint:golint
const WORD = 4                    //nolint:golint

// Frame defines new user level package format.
type Frame struct {
	// Payload, max length 4.2GB.
	payload []byte

	// Header
	header []byte
}

// ReadHeader reads only header, without payload
func ReadHeader(data []byte) *Frame {
	if lookupTable[1] == 0 {
		panic("should initialize lookup table")
	}
	//_ = data[0]
	//opt := data[0] & 0x0F
	//// if more than 2, that we have options
	//if opt > 2 {
	//	return &Frame{
	//		header:  data[:opt*WORD],
	//		payload: nil,
	//	}
	//}

	// no options
	return &Frame{
		header:  data[:8],
		payload: nil,
	}
}

func ReadFrame(data []byte) *Frame {
	if lookupTable[1] == 0 {
		panic("should initialize lookup table")
	}
	_ = data[0]
	opt := data[0] & 0x0F
	// if more than 2, that we have options
	if opt > 2 {
		return &Frame{
			header:  data[:opt*WORD],
			payload: data[opt*WORD:],
		}
	}

	// no options
	return &Frame{
		header:  data[:8],
		payload: data[8:],
	}
}

func NewFrame() *Frame {
	if lookupTable[1] == 0 {
		panic("should initialize lookup table")
	}
	f := &Frame{
		header:  make([]byte, 8),
		payload: make([]byte, 0, 100),
	}
	// set default header len (5)
	f.defaultHL()
	return f
}

// MergeHeader merge header from other frame with original payload
func (f *Frame) MergeHeader(frame *Frame) {
	f.header = frame.header
}

// To read version, we should return our 4 upper bits to their original place
// 1111_0000 -> 0000_1111 (15)
func (f *Frame) ReadVersion() byte {
	_ = f.header[0]
	return f.header[0] >> 4
}

// To write version, we should made the following:
// 1. For example we have version 15 it's 0000_1111 (1 byte)
// 2. We should shift 4 lower bits to upper and write that to the 0th byte
// 3. The 0th byte should become 1111_0000, but it's not 240, it's only 15, because version only 4 bits len
func (f *Frame) WriteVersion(version Version) {
	_ = f.header[0]
	if version > 15 {
		panic("version is only 4 bits")
	}
	f.header[0] = byte(version)<<4 | f.header[0]
}

// The lower 4 bits of the 0th octet occupies our header len data.
// We should erase upper 4 bits, which contain information about Version
// To erase, we applying bitwise AND to the upper 4 bits and returning result
func (f *Frame) readHL() byte {
	_ = f.header[0]
	// 0101_1111         0000_1111
	return f.header[0] & 0x0F
}

// Writing HL is very simple. Since we are using lower 4 bits
// we can easily apply bitwise OR and set lower 4 bits to needed hl value
func (f *Frame) writeHl(hl byte) {
	_ = f.header[0]
	f.header[0] = f.header[0] | hl
}

func (f *Frame) incrementHL() {
	_ = f.header[0]
	hl := f.readHL()
	if hl > 15 {
		panic("header len should be less than 15")
	}
	f.header[0] = f.header[0] | hl + 1
}
func (f *Frame) defaultHL() {
	_ = f.header[0]
	f.writeHl(2)
}

// Flags is full 1st byte
func (f *Frame) ReadFlags() uint8 {
	_ = f.header[1]
	return f.header[1]
}

func (f *Frame) WriteFlags(flags ...FrameFlag) {
	_ = f.header[1]
	for i := 0; i < len(flags); i++ {
		f.header[1] = f.header[1] | byte(flags[i])
	}
}

// Options slice len should not be more than 10 (40 bytes)
func (f *Frame) WriteOption(options ...uint32) {
	if options == nil {
		panic("you should write at least one option (uint32)")
	}
	hl := f.readHL()
	// check before writing. we can't handle more than 15*4 bytes of HL (2 for header and 12 for options)
	if hl == 15 {
		panic("header len could not be more than 15")
	}
	if len(options) > 10 {
		panic("header options limited by 40 bytes")
	}

	tmp := make([]byte, 0, FRAME_OPTIONS_MAX_SIZE)
	for i := 0; i < len(options); i++ {
		tmp = append(tmp, byte(options[i]))
		tmp = append(tmp, byte(options[i]>>8))
		tmp = append(tmp, byte(options[i]>>16))
		tmp = append(tmp, byte(options[i]>>24))
		f.incrementHL() // increment header len by 32 bit
	}

	f.header = append(f.header, tmp...)
}

// AppendOptions appends options to the header
func (f *Frame) AppendOptions(opts []byte) {
	f.header = append(f.header, opts...)
}

// f.readHL() - 2 needed to know actual options size
// we know, that 2 WORDS is minimal header len
// extra WORDS will add extra 32bits to the options (4 bytes)
//
func (f *Frame) ReadOptions() []uint32 {
	// we can read options, if there are no options
	if f.readHL() <= 2 {
		return nil
	}
	// Get the options len
	optionLen := f.readHL() - 2 // 2 is the default
	// slice in place
	options := make([]uint32, 0, optionLen)

	// Options starting from 8-th byte
	// we should scan with 4 byte window (32bit, WORD)
	for i := byte(0); i < optionLen*WORD; i += WORD {
		// for example
		// 8  12  16
		// 9  13  17
		// 10 14  18
		// 11 15  19
		// For this data, HL will be 3, optionLen will be 12 (3*4) bytes
		options = append(options, uint32(f.header[8+i])|uint32(f.header[8+i+1])<<8|uint32(f.header[8+i+2])<<16|uint32(f.header[8+i+3])<<24)
	}
	return options
}

// LE format used to write Payload
// Using 4 bytes (2,3,4,5 bytes in the header)
func (f *Frame) ReadPayloadLen() uint32 {
	// 2,3,4,5
	_ = f.header[5]
	return uint32(f.header[2]) | uint32(f.header[3])<<8 | uint32(f.header[4])<<16 | uint32(f.header[5])<<24
}

// LE format used to write Payload
// Using 4 bytes (2,3,4,5 bytes in the header)
func (f *Frame) WritePayloadLen(len uint32) {
	_ = f.header[5]
	f.header[2] = byte(len)
	f.header[3] = byte(len >> 8)
	f.header[4] = byte(len >> 16)
	f.header[5] = byte(len >> 24)
}

// Calculating CRC and writing it to the 6th byte (7th reserved)
func (f *Frame) WriteCRC() {
	_ = f.header[7]
	crc := byte(0)
	hl := f.readHL()
	// write CRC with options
	if f.readHL() > 2 {
		for i := byte(0); i < hl*WORD; i++ {
			data := f.header[i] ^ crc
			crc = lookupTable[data]
		}
		f.header[6] = crc
		return
	}

	for i := 0; i < 6; i++ {
		data := f.header[i] ^ crc
		crc = lookupTable[data]
	}

	f.header[6] = crc
}

// Reading info from 6th byte and verifying it with calculated in-place. Should be equal.
// If not - drop the frame as incorrect.
func (f *Frame) VerifyCRC() bool {
	_ = f.header[7]
	crc := byte(0)
	hl := f.readHL()
	if hl > 2 {
		for i := byte(0); i < hl*WORD; i++ {
			// to verify, we are skipping the CRC field itself
			if i == 6 {
				data := 0 ^ crc
				crc = lookupTable[data]
				continue
			}
			data := f.header[i] ^ crc
			crc = lookupTable[data]
		}
		return crc == f.header[6]
	}

	for i := 0; i < 6; i++ {
		data := f.header[i] ^ crc
		crc = lookupTable[data]
	}
	return crc == f.header[6]
}

// Bytes returns header with payload
func (f *Frame) Bytes() []byte {
	buf := make([]byte, 0, len(f.header)+len(f.payload))
	buf = append(buf, f.header...)
	buf = append(buf, f.payload...)
	return buf
}

// Header returns frame header
func (f *Frame) Header() []byte {
	return f.header
}

// Payload returns frame payload without header
func (f *Frame) Payload() []byte {
	// start from the 1st (staring from 0) byte
	return f.payload
}

//
func (f *Frame) WritePayload(data []byte) {
	f.payload = make([]byte, len(data))
	copy(f.payload, data)
}

//go:nosplit
//go:nocheckptr
func noescape(p unsafe.Pointer) unsafe.Pointer {
	x := uintptr(p)
	return unsafe.Pointer(x ^ 0) //nolint:staticcheck
}

// After reset you should write all data from the start
func (f *Frame) Reset() {
	f.header = make([]byte, 0, 8)
	f.payload = make([]byte, 0, 100)
}
<|endoftext|>


<commit_before>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"bytes"
)

const (
	FrameHeaderSize = 10
)

type FrameType struct {
	id          string
	description string
	constructor func(FrameHead, []byte) Framer
}

type Framer interface {
	Id() string
	Size() int
	String() string
}

type FrameHead struct {
	FrameType
	statusFlags byte
	formatFlags byte
	size        int32
}

func (h FrameHead) Id() string {
	return h.id
}

func (h FrameHead) Size() int {
	return int(h.size)
}

type DataFrame struct {
	FrameHead
	Data []byte
}

func NewDataFrame(head FrameHead, data []byte) Framer {
	return &DataFrame{head, data}
}

func (f DataFrame) String() string {
	return "<binary data>"
}

type TextFrame struct {
	FrameHead
	Encoding string
	Text     string
}

func NewTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &TextFrame{FrameHead: head}

	encodingIndex := data[0]
	f.Encoding = encodingForIndex(data[0])

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[1:])); err != nil {
		return nil
	}

	return f
}

func (f TextFrame) String() string {
	return f.Text
}

type DescTextFrame struct {
	FrameHead
	TextFrame
	Description string
}

func NewDescTextFrame(head FrameHead, data []byte) Framer {
	f := &DescTextFrame{FrameHead: head}

	var err error
	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)

	cutoff := 1
	if i := afterNullIndex(data[1:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[1:cutoff])); err != nil {
		return nil
	}

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

type UnsynchTextFrame struct {
	FrameHead
	DescTextFrame
	Language          string
	ContentDescriptor string
}

func NewUnsynchTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &UnsynchTextFrame{FrameHead: head}

	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)
	f.Language = string(data[1:4])

	cutoff := 4
	if i := afterNullIndex(data[4:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[4:cutoff])); err != nil {
		return nil
	}

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

type ImageFrame struct {
	FrameHead
	DataFrame
	Encoding    string
	MIMEType    string
	PictureType byte
	Description string
}

func NewImageFrame(head FrameHead, data []byte) Framer {
	f := &ImageFrame{FrameHead: head}

	var err error
	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)

	buffer := bytes.NewBuffer(data[1:])
	if f.MIMEType, err = buffer.ReadString(0); err != nil {
		return nil
	}

	if f.PictureType, err = buffer.ReadByte(); err != nil {
		return nil
	}

	beginIndex := 1 + len(f.MIMEType) + 1
	var cutoff int
	if i := afterNullIndex(data[beginIndex:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff = beginIndex + i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[beginIndex:cutoff])); err != nil {
		return nil
	}

	f.Data = data[cutoff:]

	return f
}
<commit_msg>Add framer to []byte methods<commit_after>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"bytes"
)

const (
	FrameHeaderSize = 10
)

type FrameType struct {
	id          string
	description string
	constructor func(FrameHead, []byte) Framer
}

type Framer interface {
	Id() string
	Size() int
	String() string
	Bytes() []byte
}

type FrameHead struct {
	FrameType
	statusFlags byte
	formatFlags byte
	size        int32
}

func (h FrameHead) Id() string {
	return h.id
}

func (h FrameHead) Size() int {
	return int(h.size)
}

func (h FrameHead) Bytes() []byte {
	bytes := make([]byte, FrameHeaderSize)

	copy(bytes[:4], []byte(h.id))
	copy(bytes[4:8], intbytes(h.size, NormByteLength))
	bytes[8] = h.statusFlags
	bytes[9] = h.formatFlags

	return bytes
}

type DataFrame struct {
	FrameHead
	Data []byte
}

func NewDataFrame(head FrameHead, data []byte) Framer {
	return &DataFrame{head, data}
}

func (f DataFrame) String() string {
	return "<binary data>"
}

func (f DataFrame) Bytes() []byte {
	return append(f.FrameHead.Bytes(), f.Data...)
}

type TextFrame struct {
	FrameHead
	Encoding string
	Text     string
}

func NewTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &TextFrame{FrameHead: head}

	encodingIndex := data[0]
	f.Encoding = encodingForIndex(data[0])

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[1:])); err != nil {
		return nil
	}

	return f
}

func (f TextFrame) String() string {
	return f.Text
}

func (f TextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodingIndex := indexForEncoding(f.Encoding)
	encodedString, err := Encoders[encodingIndex].ConvertString(f.Text)
	if err != nil {
		return bytes
	}

	bytes[0] = encodingIndex
	copy(bytes[1:], []byte(encodedString))

	return append(f.FrameHead.Bytes(), bytes...)
}

type DescTextFrame struct {
	FrameHead
	TextFrame
	Description string
}

func NewDescTextFrame(head FrameHead, data []byte) Framer {
	f := &DescTextFrame{FrameHead: head}

	var err error
	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)

	cutoff := 1
	if i := afterNullIndex(data[1:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[1:cutoff])); err != nil {
		return nil
	}

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

func (f DescTextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodingIndex := indexForEncoding(f.Encoding)
	encodedDescription, err := Encoders[encodingIndex].ConvertString(f.Description)
	if err != nil {
		return bytes
	}
	encodedText, err := Encoders[encodingIndex].ConvertString(f.Text)
	if err != nil {
		return bytes
	}

	bytes[0] = encodingIndex
	index := 1
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(encodedText)], []byte(encodedText))

	return append(f.FrameHead.Bytes(), bytes...)
}

type UnsynchTextFrame struct {
	FrameHead
	DescTextFrame
	Language          string
	ContentDescriptor string
}

func NewUnsynchTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &UnsynchTextFrame{FrameHead: head}

	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)
	f.Language = string(data[1:4])

	cutoff := 4
	if i := afterNullIndex(data[4:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[4:cutoff])); err != nil {
		return nil
	}

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}

func (f UnsynchTextFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodingIndex := indexForEncoding(f.Encoding)
	encodedDescription, err := Encoders[encodingIndex].ConvertString(f.Description)
	if err != nil {
		return bytes
	}
	encodedText, err := Encoders[encodingIndex].ConvertString(f.Text)
	if err != nil {
		return bytes
	}

	bytes[0] = encodingIndex
	copy(bytes[1:4], []byte(f.Language))
	index := 4
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(encodedText)], []byte(encodedText))

	return append(f.FrameHead.Bytes(), bytes...)
}

type ImageFrame struct {
	FrameHead
	DataFrame
	Encoding    string
	MIMEType    string
	PictureType byte
	Description string
}

func NewImageFrame(head FrameHead, data []byte) Framer {
	f := &ImageFrame{FrameHead: head}

	var err error
	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)

	buffer := bytes.NewBuffer(data[1:])
	if f.MIMEType, err = buffer.ReadString(0); err != nil {
		return nil
	}

	if f.PictureType, err = buffer.ReadByte(); err != nil {
		return nil
	}

	beginIndex := 1 + len(f.MIMEType) + 1
	var cutoff int
	if i := afterNullIndex(data[beginIndex:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff = beginIndex + i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[beginIndex:cutoff])); err != nil {
		return nil
	}

	f.Data = data[cutoff:]

	return f
}

func (f ImageFrame) Bytes() []byte {
	bytes := make([]byte, f.Size())

	encodingIndex := indexForEncoding(f.Encoding)
	encodedDescription, err := Encoders[encodingIndex].ConvertString(f.Description)
	if err != nil {
		return bytes
	}

	bytes[0] = encodingIndex
	index := 1
	copy(bytes[index:index+len(f.MIMEType)], []byte(f.MIMEType))
	index += len(f.MIMEType)
	bytes[index] = f.PictureType
	index += 1
	copy(bytes[index:index+len(encodedDescription)], []byte(encodedDescription))
	index += len(encodedDescription)
	copy(bytes[index:index+len(f.Data)], f.Data)

	return append(f.FrameHead.Bytes(), bytes...)
}
<|endoftext|>


<commit_before>package fs

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)

// Object represents a locally stored LFS object.
type Object struct {
	Oid  string
	Size int64
}

type Filesystem struct {
	GitStorageDir string   // parent of objects/lfs (may be same as GitDir but may not)
	LFSStorageDir string   // parent of lfs objects and tmp dirs. Default: ".git/lfs"
	ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
	lfsobjdir     string
	tmpdir        string
	logdir        string
	mu            sync.Mutex
}

func (f *Filesystem) EachObject(fn func(Object) error) error {
	var eachErr error
	tools.FastWalkGitRepo(f.LFSObjectDir(), func(parentDir string, info os.FileInfo, err error) {
		if err != nil {
			eachErr = err
			return
		}
		if eachErr != nil || info.IsDir() {
			return
		}
		if oidRE.MatchString(info.Name()) {
			fn(Object{Oid: info.Name(), Size: info.Size()})
		}
	})
	return eachErr
}

func (f *Filesystem) ObjectExists(oid string, size int64) bool {
	return tools.FileExistsOfSize(f.ObjectPathname(oid), size)
}

func (f *Filesystem) ObjectPath(oid string) (string, error) {
	dir := f.localObjectDir(oid)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", fmt.Errorf("Error trying to create local storage directory in %q: %s", dir, err)
	}
	return filepath.Join(dir, oid), nil
}

func (f *Filesystem) ObjectPathname(oid string) string {
	return filepath.Join(f.localObjectDir(oid), oid)
}

func (f *Filesystem) DecodePathname(path string) string {
	return string(DecodePathBytes([]byte(path)))
}

/**
 * Revert non ascii chracters escaped by git or windows (as octal sequences \000) back to bytes.
 */
func DecodePathBytes(path []byte) []byte {
	var expression = regexp.MustCompile(`\\[0-9]{3}`)
	var buffer bytes.Buffer

	// strip quotes if any
	if len(path) > 2 && path[0] == '"' && path[len(path)-1] == '"' {
		path = path[1 : len(path)-1]
	}

	base := 0
	for _, submatches := range expression.FindAllSubmatchIndex(path, -1) {
		buffer.Write(path[base:submatches[0]])

		match := string(path[submatches[0]+1 : submatches[0]+4])

		k, err := strconv.ParseUint(match, 8, 64)
		if err != nil {
			return path
		} // abort on error

		buffer.Write([]byte{byte(k)})
		base = submatches[1]
	}

	buffer.Write(path[base:len(path)])

	return buffer.Bytes()
}

func (f *Filesystem) localObjectDir(oid string) string {
	return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
}

func (f *Filesystem) ObjectReferencePaths(oid string) []string {
	if len(f.ReferenceDirs) == 0 {
		return nil
	}

	var paths []string
	for _, ref := range f.ReferenceDirs {
		paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
	}
	return paths
}

func (f *Filesystem) LFSObjectDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.lfsobjdir) == 0 {
		f.lfsobjdir = filepath.Join(f.LFSStorageDir, "objects")
		os.MkdirAll(f.lfsobjdir, 0755)
	}

	return f.lfsobjdir
}

func (f *Filesystem) LogDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.logdir) == 0 {
		f.logdir = filepath.Join(f.LFSStorageDir, "logs")
		os.MkdirAll(f.logdir, 0755)
	}

	return f.logdir
}

func (f *Filesystem) TempDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.tmpdir) == 0 {
		f.tmpdir = filepath.Join(f.LFSStorageDir, "tmp")
		os.MkdirAll(f.tmpdir, 0755)
	}

	return f.tmpdir
}

func (f *Filesystem) Cleanup() error {
	if f == nil {
		return nil
	}
	return f.cleanupTmp()
}

// New initializes a new *Filesystem with the given directories. gitdir is the
// path to the bare repo, workdir is the path to the repository working
// directory, and lfsdir is the optional path to the `.git/lfs` directory.
func New(gitdir, workdir, lfsdir string) *Filesystem {
	fs := &Filesystem{
		GitStorageDir: resolveGitStorageDir(gitdir),
	}

	fs.ReferenceDirs = resolveReferenceDirs(fs.GitStorageDir)

	if len(lfsdir) == 0 {
		lfsdir = "lfs"
	}

	if filepath.IsAbs(lfsdir) {
		fs.LFSStorageDir = lfsdir
	} else {
		fs.LFSStorageDir = filepath.Join(fs.GitStorageDir, lfsdir)
	}

	return fs
}

func resolveReferenceDirs(gitStorageDir string) []string {
	var references []string

	cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
	if tools.FileExists(cloneReferencePath) {
		f, err := os.Open(cloneReferencePath)
		if err != nil {
			tracerx.Printf("could not open %s: %s",
				cloneReferencePath, err)
			return nil
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
	L:
		for scanner.Scan() {
			text := strings.TrimSpace(scanner.Text())
			if len(text) == 0 {
				continue
			}

			var objs string
			switch text[0] {
			case '#':
				continue L
			case '"':
				unquote := strings.LastIndex(text, "\"")
				if unquote == 0 {
					continue L
				}

				objs, _ = strconv.Unquote(text[:unquote+1])
			default:
				objs = text
			}
			storage := filepath.Join(filepath.Dir(objs),
				"lfs", "objects")

			if tools.DirExists(storage) {
				references = append(references, storage)
			}
		}

		if err := scanner.Err(); err != nil {
			tracerx.Printf("could not scan %s: %s",
				cloneReferencePath, err)
		}
	}
	return references
}

// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
// are found in another git dir via 'commondir'.
func resolveGitStorageDir(gitDir string) string {
	commondirpath := filepath.Join(gitDir, "commondir")
	if tools.FileExists(commondirpath) && !tools.DirExists(filepath.Join(gitDir, "objects")) {
		// no git-dir: prefix in commondir
		storage, err := processGitRedirectFile(commondirpath, "")
		if err == nil {
			return storage
		}
	}
	return gitDir
}

func processGitRedirectFile(file, prefix string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}

	contents := string(data)
	var dir string
	if len(prefix) > 0 {
		if !strings.HasPrefix(contents, prefix) {
			// Prefix required & not found
			return "", nil
		}
		dir = strings.TrimSpace(contents[len(prefix):])
	} else {
		dir = strings.TrimSpace(contents)
	}

	if !filepath.IsAbs(dir) {
		// The .git file contains a relative path.
		// Create an absolute path based on the directory the .git file is located in.
		dir = filepath.Join(filepath.Dir(file), dir)
	}

	return dir, nil
}
<commit_msg>fs/fs.go: extract func existsAlternate()<commit_after>package fs

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)

// Object represents a locally stored LFS object.
type Object struct {
	Oid  string
	Size int64
}

type Filesystem struct {
	GitStorageDir string   // parent of objects/lfs (may be same as GitDir but may not)
	LFSStorageDir string   // parent of lfs objects and tmp dirs. Default: ".git/lfs"
	ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
	lfsobjdir     string
	tmpdir        string
	logdir        string
	mu            sync.Mutex
}

func (f *Filesystem) EachObject(fn func(Object) error) error {
	var eachErr error
	tools.FastWalkGitRepo(f.LFSObjectDir(), func(parentDir string, info os.FileInfo, err error) {
		if err != nil {
			eachErr = err
			return
		}
		if eachErr != nil || info.IsDir() {
			return
		}
		if oidRE.MatchString(info.Name()) {
			fn(Object{Oid: info.Name(), Size: info.Size()})
		}
	})
	return eachErr
}

func (f *Filesystem) ObjectExists(oid string, size int64) bool {
	return tools.FileExistsOfSize(f.ObjectPathname(oid), size)
}

func (f *Filesystem) ObjectPath(oid string) (string, error) {
	dir := f.localObjectDir(oid)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", fmt.Errorf("Error trying to create local storage directory in %q: %s", dir, err)
	}
	return filepath.Join(dir, oid), nil
}

func (f *Filesystem) ObjectPathname(oid string) string {
	return filepath.Join(f.localObjectDir(oid), oid)
}

func (f *Filesystem) DecodePathname(path string) string {
	return string(DecodePathBytes([]byte(path)))
}

/**
 * Revert non ascii chracters escaped by git or windows (as octal sequences \000) back to bytes.
 */
func DecodePathBytes(path []byte) []byte {
	var expression = regexp.MustCompile(`\\[0-9]{3}`)
	var buffer bytes.Buffer

	// strip quotes if any
	if len(path) > 2 && path[0] == '"' && path[len(path)-1] == '"' {
		path = path[1 : len(path)-1]
	}

	base := 0
	for _, submatches := range expression.FindAllSubmatchIndex(path, -1) {
		buffer.Write(path[base:submatches[0]])

		match := string(path[submatches[0]+1 : submatches[0]+4])

		k, err := strconv.ParseUint(match, 8, 64)
		if err != nil {
			return path
		} // abort on error

		buffer.Write([]byte{byte(k)})
		base = submatches[1]
	}

	buffer.Write(path[base:len(path)])

	return buffer.Bytes()
}

func (f *Filesystem) localObjectDir(oid string) string {
	return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
}

func (f *Filesystem) ObjectReferencePaths(oid string) []string {
	if len(f.ReferenceDirs) == 0 {
		return nil
	}

	var paths []string
	for _, ref := range f.ReferenceDirs {
		paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
	}
	return paths
}

func (f *Filesystem) LFSObjectDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.lfsobjdir) == 0 {
		f.lfsobjdir = filepath.Join(f.LFSStorageDir, "objects")
		os.MkdirAll(f.lfsobjdir, 0755)
	}

	return f.lfsobjdir
}

func (f *Filesystem) LogDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.logdir) == 0 {
		f.logdir = filepath.Join(f.LFSStorageDir, "logs")
		os.MkdirAll(f.logdir, 0755)
	}

	return f.logdir
}

func (f *Filesystem) TempDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.tmpdir) == 0 {
		f.tmpdir = filepath.Join(f.LFSStorageDir, "tmp")
		os.MkdirAll(f.tmpdir, 0755)
	}

	return f.tmpdir
}

func (f *Filesystem) Cleanup() error {
	if f == nil {
		return nil
	}
	return f.cleanupTmp()
}

// New initializes a new *Filesystem with the given directories. gitdir is the
// path to the bare repo, workdir is the path to the repository working
// directory, and lfsdir is the optional path to the `.git/lfs` directory.
func New(gitdir, workdir, lfsdir string) *Filesystem {
	fs := &Filesystem{
		GitStorageDir: resolveGitStorageDir(gitdir),
	}

	fs.ReferenceDirs = resolveReferenceDirs(fs.GitStorageDir)

	if len(lfsdir) == 0 {
		lfsdir = "lfs"
	}

	if filepath.IsAbs(lfsdir) {
		fs.LFSStorageDir = lfsdir
	} else {
		fs.LFSStorageDir = filepath.Join(fs.GitStorageDir, lfsdir)
	}

	return fs
}

func resolveReferenceDirs(gitStorageDir string) []string {
	var references []string

	cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
	if tools.FileExists(cloneReferencePath) {
		f, err := os.Open(cloneReferencePath)
		if err != nil {
			tracerx.Printf("could not open %s: %s",
				cloneReferencePath, err)
			return nil
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
	L:
		for scanner.Scan() {
			text := strings.TrimSpace(scanner.Text())
			if len(text) == 0 || strings.StartsWith(text, "#") {
				continue
			}

			if dir, ok := existsAlternate(text); ok {
				references = append(references, dir)
			}
		}

		if err := scanner.Err(); err != nil {
			tracerx.Printf("could not scan %s: %s",
				cloneReferencePath, err)
		}
	}
	return references
}

// existsAlternate takes an object directory given in "objs" (read as a single,
// line from .git/objects/info/alternates). If that is a satisfiable alternates
// directory (i.e., it exists), the directory is returned along with "true". If
// not, the empty string and false is returned instead.
func existsAlternate(objs string) (string, bool) {
	objs = strings.TrimSpace(objs)
	if objs.StartsWith(objs, "#") {
		var err error

		unquote := strings.LastIndex(text, "\"")
		if unquote == 0 {
			continue L
		}

		objs, err = strconv.Unquote(text[:unquote+1])
		if err != nil {
			return "", false
		}
	}

	storage := filepath.Join(filepath.Dir(objs), "lfs", "objects")

	if tools.DirExists(storage) {
		return storage, true
	}
	return "", false
}

// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
// are found in another git dir via 'commondir'.
func resolveGitStorageDir(gitDir string) string {
	commondirpath := filepath.Join(gitDir, "commondir")
	if tools.FileExists(commondirpath) && !tools.DirExists(filepath.Join(gitDir, "objects")) {
		// no git-dir: prefix in commondir
		storage, err := processGitRedirectFile(commondirpath, "")
		if err == nil {
			return storage
		}
	}
	return gitDir
}

func processGitRedirectFile(file, prefix string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}

	contents := string(data)
	var dir string
	if len(prefix) > 0 {
		if !strings.HasPrefix(contents, prefix) {
			// Prefix required & not found
			return "", nil
		}
		dir = strings.TrimSpace(contents[len(prefix):])
	} else {
		dir = strings.TrimSpace(contents)
	}

	if !filepath.IsAbs(dir) {
		// The .git file contains a relative path.
		// Create an absolute path based on the directory the .git file is located in.
		dir = filepath.Join(filepath.Dir(file), dir)
	}

	return dir, nil
}
<|endoftext|>


<commit_before>package topgun_test

import (
	"time"

	_ "github.com/lib/pq"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Worker failing", func() {
	BeforeEach(func() {
		Deploy("deployments/concourse-different-workers.yml", "-o", "operations/other-worker-tagged.yml", "-o", "operations/fast-gc.yml")
	})

	Context("when baggageclaim becomes unresponsive", func() {
		BeforeEach(func() {
			By("setting a pipeline that uses the doomed worker")
			fly("set-pipeline", "-n", "-c", "pipelines/controlled-timer-doomed-worker.yml", "-p", "worker-failing-test")
			fly("unpause-pipeline", "-p", "worker-failing-test")

			By("running the build on the doomed worker")
			fly("trigger-job", "-w", "-j", "worker-failing-test/use-doomed-worker")

			By("making baggageclaim become unresponsive on the doomed worker")
			bosh("ssh", "other_worker/0", "-c", "sudo pkill -F /var/vcap/sys/run/baggageclaim/baggageclaim.pid -STOP")

			By("running check-resource to force the existing volume to be no longer desired")
			fly("check-resource", "-r", "worker-failing-test/controlled-timer")

			// at this point, GC should try to remove the volume, and begin to hang
		})

		AfterEach(func() {
			bosh("ssh", "other_worker/0", "-c", "sudo pkill -F /var/vcap/sys/run/baggageclaim/baggageclaim.pid -CONT")
			waitForWorkersToBeRunning()
		})

		It("puts the worker in stalled state and does not lock up garbage collection", func() {
			By("waiting for the doomed worker to stall")
			Eventually(waitForStalledWorker()).ShouldNot(BeEmpty())

			By("running the build on the safe worker")
			fly("trigger-job", "-w", "-j", "worker-failing-test/use-safe-worker")

			By("having a cache for the controlled-timer resource")
			Expect(volumesByResourceType("time")).ToNot(BeEmpty())

			By("waiting long enough for the resource to have a new version available upon check")
			time.Sleep(5 * time.Second)

			By("running check-resource to force the existing volume on the safe worker to be no longer desired")
			fly("check-resource", "-r", "worker-failing-test/controlled-timer")

			By("eventually garbage collecting the volume from the safe worker")
			Eventually(func() []string {
				return volumesByResourceType("time")
			}).Should(BeEmpty())
		})
	})
})
<commit_msg>fix ops file<commit_after>package topgun_test

import (
	"time"

	_ "github.com/lib/pq"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Worker failing", func() {
	BeforeEach(func() {
		Deploy("deployments/concourse-different-workers.yml", "-o", "operations/other-worker-doomed.yml", "-o", "operations/fast-gc.yml")
	})

	Context("when baggageclaim becomes unresponsive", func() {
		BeforeEach(func() {
			By("setting a pipeline that uses the doomed worker")
			fly("set-pipeline", "-n", "-c", "pipelines/controlled-timer-doomed-worker.yml", "-p", "worker-failing-test")
			fly("unpause-pipeline", "-p", "worker-failing-test")

			By("running the build on the doomed worker")
			fly("trigger-job", "-w", "-j", "worker-failing-test/use-doomed-worker")

			By("making baggageclaim become unresponsive on the doomed worker")
			bosh("ssh", "other_worker/0", "-c", "sudo pkill -F /var/vcap/sys/run/baggageclaim/baggageclaim.pid -STOP")

			By("running check-resource to force the existing volume to be no longer desired")
			fly("check-resource", "-r", "worker-failing-test/controlled-timer")

			// at this point, GC should try to remove the volume, and begin to hang
		})

		AfterEach(func() {
			bosh("ssh", "other_worker/0", "-c", "sudo pkill -F /var/vcap/sys/run/baggageclaim/baggageclaim.pid -CONT")
			waitForWorkersToBeRunning()
		})

		It("puts the worker in stalled state and does not lock up garbage collection", func() {
			By("waiting for the doomed worker to stall")
			Eventually(waitForStalledWorker()).ShouldNot(BeEmpty())

			By("running the build on the safe worker")
			fly("trigger-job", "-w", "-j", "worker-failing-test/use-safe-worker")

			By("having a cache for the controlled-timer resource")
			Expect(volumesByResourceType("time")).ToNot(BeEmpty())

			By("waiting long enough for the resource to have a new version available upon check")
			time.Sleep(5 * time.Second)

			By("running check-resource to force the existing volume on the safe worker to be no longer desired")
			fly("check-resource", "-r", "worker-failing-test/controlled-timer")

			By("eventually garbage collecting the volume from the safe worker")
			Eventually(func() []string {
				return volumesByResourceType("time")
			}).Should(BeEmpty())
		})
	})
})
<|endoftext|>