<commit_before>/*
Package rest contains the company standard REST API client implementation.

This package can be used with any type that implements the Consumer interface:

    rc := rest.Client{myConsumer{
    	"Key",
    	"Secret",
    }}

    params := map[string]string{
        "something": "cool"
    }

    resp, err := rc.GetJson("http://example.com/api/", params, nil)
*/
package rest

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Supported HTTP Methods
const (
	DELETE = "DELETE"
	GET    = "GET"
	POST   = "POST"
	PUT    = "PUT"
)

// Consumer encapsulates the required authentication logic for the API the
// Client will interact with.
type Consumer interface {
	AuthorizeRequest(urlStr string, requestType string, form url.Values) url.Values
}

// Client is the type that encapsulates and uses the Consumer to sign any REST
// requests that are performed.
type Client struct {
	Consumer
}

// BaseResponse is the resultant type of any of the Do*() methods of the
// Client. It contains several informational fields as well as the result
// value.
type BaseResponse struct {
	Status   int         // HTTP status code.
	Reason   string      // Reason message for the status code.
	Messages []string    // Any server messages attached to the Response.
	Next     string      // A cursor to the next result set.
	Results  interface{} // The actual results of the REST request.
}

func (c *Client) decode(r io.Reader, want interface{}) (*BaseResponse, error) {
	resp := &BaseResponse{Results: want}
	if err := json.NewDecoder(r).Decode(resp); err != nil {
		return resp, err
	}
	return resp, nil
}

// BuildForm will take a map of the form input and build a url.Values object.
func (c *Client) BuildForm(params map[string]string) url.Values {
	form := url.Values{}

	for param, value := range params {
		form.Set(param, value)
	}

	return form
}

func (c *Client) do(method, urlStr string, params map[string]string) (*http.Response, error) {
	req, err := http.NewRequest(method, urlStr, nil)

	if err != nil {
		return nil, err
	}

	form := c.AuthorizeRequest(urlStr, method, c.BuildForm(params))

	// TODO: Investigate if this needs to be different for POST
	req.URL.RawQuery = form.Encode()

	return http.DefaultClient.Do(req)
}

func (c *Client) doJson(method, url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	response, err := c.do(method, url, params)

	if err != nil {
		return &BaseResponse{}, err
	}

	defer response.Body.Close()

	b, _ := ioutil.ReadAll(response.Body)
	resp, e := c.decode(strings.NewReader(string(b)), entity)

	return resp, e
}

// Get will perform a HTTP GET against the supplied URL with the given parameters.
func (c *Client) Get(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(GET, urlStr, params)
}

// GetJson will perform a HTTP GET and will JSON decode the response.
func (c *Client) GetJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(GET, url, params, entity)
}

// Post will perform a HTTP POST against the supplied URL with the given parameters.
func (c *Client) Post(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(POST, urlStr, params)
}

// PostJson will perform a HTTP POST and will JSON decode the response.
func (c *Client) PostJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(POST, url, params, entity)
}

// Put will perform a HTTP PUT against the supplied URL with the given parameters.
func (c *Client) Put(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(PUT, urlStr, params)
}

// PutJson will perform a HTTP PUT and will JSON decode the response.
func (c *Client) PutJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(PUT, url, params, entity)
}

// Delete will perform a HTTP DELETE against the supplied URL with the given parameters.
func (c *Client) Delete(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(DELETE, urlStr, params)
}

// DeleteJson will perform a HTTP DELETE and will JSON decode the response.
func (c *Client) DeleteJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(DELETE, url, params, entity)
}
<commit_msg>Return the error from Decode()<commit_after>/*
Package rest contains the company standard REST API client implementation.

This package can be used with any type that implements the Consumer interface:

    rc := rest.Client{myConsumer{
    	"Key",
    	"Secret",
    }}

    params := map[string]string{
        "something": "cool"
    }

    resp, err := rc.GetJson("http://example.com/api/", params, nil)
*/
package rest

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Supported HTTP Methods
const (
	DELETE = "DELETE"
	GET    = "GET"
	POST   = "POST"
	PUT    = "PUT"
)

// Consumer encapsulates the required authentication logic for the API the
// Client will interact with.
type Consumer interface {
	AuthorizeRequest(urlStr string, requestType string, form url.Values) url.Values
}

// Client is the type that encapsulates and uses the Consumer to sign any REST
// requests that are performed.
type Client struct {
	Consumer
}

// BaseResponse is the resultant type of any of the Do*() methods of the
// Client. It contains several informational fields as well as the result
// value.
type BaseResponse struct {
	Status   int         // HTTP status code.
	Reason   string      // Reason message for the status code.
	Messages []string    // Any server messages attached to the Response.
	Next     string      // A cursor to the next result set.
	Results  interface{} // The actual results of the REST request.
}

func (c *Client) decode(r io.Reader, want interface{}) (*BaseResponse, error) {
	resp := &BaseResponse{Results: want}
	err := json.NewDecoder(r).Decode(resp)
	return resp, err
}

// BuildForm will take a map of the form input and build a url.Values object.
func (c *Client) BuildForm(params map[string]string) url.Values {
	form := url.Values{}

	for param, value := range params {
		form.Set(param, value)
	}

	return form
}

func (c *Client) do(method, urlStr string, params map[string]string) (*http.Response, error) {
	req, err := http.NewRequest(method, urlStr, nil)

	if err != nil {
		return nil, err
	}

	form := c.AuthorizeRequest(urlStr, method, c.BuildForm(params))

	// TODO: Investigate if this needs to be different for POST
	req.URL.RawQuery = form.Encode()

	return http.DefaultClient.Do(req)
}

func (c *Client) doJson(method, url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	response, err := c.do(method, url, params)
	if err != nil {
		return &BaseResponse{}, err
	}

	defer response.Body.Close()

	b, _ := ioutil.ReadAll(response.Body)
	return c.decode(strings.NewReader(string(b)), entity)
}

// Get will perform a HTTP GET against the supplied URL with the given parameters.
func (c *Client) Get(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(GET, urlStr, params)
}

// GetJson will perform a HTTP GET and will JSON decode the response.
func (c *Client) GetJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(GET, url, params, entity)
}

// Post will perform a HTTP POST against the supplied URL with the given parameters.
func (c *Client) Post(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(POST, urlStr, params)
}

// PostJson will perform a HTTP POST and will JSON decode the response.
func (c *Client) PostJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(POST, url, params, entity)
}

// Put will perform a HTTP PUT against the supplied URL with the given parameters.
func (c *Client) Put(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(PUT, urlStr, params)
}

// PutJson will perform a HTTP PUT and will JSON decode the response.
func (c *Client) PutJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(PUT, url, params, entity)
}

// Delete will perform a HTTP DELETE against the supplied URL with the given parameters.
func (c *Client) Delete(urlStr string, params map[string]string) (*http.Response, error) {
	return c.do(DELETE, urlStr, params)
}

// DeleteJson will perform a HTTP DELETE and will JSON decode the response.
func (c *Client) DeleteJson(url string, params map[string]string, entity interface{}) (*BaseResponse, error) {
	return c.doJson(DELETE, url, params, entity)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2015 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package importx

import (
	"encoding/json"
	"flag"
	"os"

	"golang.org/x/net/context"

	"github.com/vmware/govmomi/ovf"
	"github.com/vmware/govmomi/vim25/types"
)

type Property struct {
	types.KeyValue
	Spec *ovf.Property `json:",omitempty"`
}

type Network struct {
	Name    string
	Network string
}

type Options struct {
	AllDeploymentOptions []string `json:",omitempty"`
	Deployment           string

	AllDiskProvisioningOptions []string `json:",omitempty"`
	DiskProvisioning           string

	AllIPAllocationPolicyOptions []string `json:",omitempty"`
	IPAllocationPolicy           string

	AllIPProtocolOptions []string `json:",omitempty"`
	IPProtocol           string

	PropertyMapping []Property `json:",omitempty"`

	NetworkMapping []Network `json:",omitempty"`

	Annotation string `json:",omitempty"`

	PowerOn      bool
	InjectOvfEnv bool
	WaitForIP    bool
	Name         *string
}

type OptionsFlag struct {
	Options Options

	path string
}

func newOptionsFlag(ctx context.Context) (*OptionsFlag, context.Context) {
	return &OptionsFlag{}, ctx
}

func (flag *OptionsFlag) Register(ctx context.Context, f *flag.FlagSet) {
	f.StringVar(&flag.path, "options", "", "Options spec file path for VM deployment")
}

func (flag *OptionsFlag) Process(ctx context.Context) error {
	if len(flag.path) > 0 {
		f, err := os.Open(flag.path)
		if err != nil {
			return err
		}
		defer f.Close()

		if err := json.NewDecoder(f).Decode(&flag.Options); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>govc - add options.json to importx commands<commit_after>/*
Copyright (c) 2015 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package importx

import (
	"encoding/json"
	"errors"
	"flag"
	"io"
	"os"
	"strings"

	"golang.org/x/net/context"

	"github.com/vmware/govmomi/ovf"
	"github.com/vmware/govmomi/vim25/types"
)

type Property struct {
	types.KeyValue
	Spec *ovf.Property `json:",omitempty"`
}

type Network struct {
	Name    string
	Network string
}

type Options struct {
	AllDeploymentOptions []string `json:",omitempty"`
	Deployment           string

	AllDiskProvisioningOptions []string `json:",omitempty"`
	DiskProvisioning           string

	AllIPAllocationPolicyOptions []string `json:",omitempty"`
	IPAllocationPolicy           string

	AllIPProtocolOptions []string `json:",omitempty"`
	IPProtocol           string

	PropertyMapping []Property `json:",omitempty"`

	NetworkMapping []Network `json:",omitempty"`

	Annotation string `json:",omitempty"`

	PowerOn      bool
	InjectOvfEnv bool
	WaitForIP    bool
	Name         *string
}

type OptionsFlag struct {
	Options Options

	path string
	json string
}

func newOptionsFlag(ctx context.Context) (*OptionsFlag, context.Context) {
	return &OptionsFlag{}, ctx
}

func (flag *OptionsFlag) Register(ctx context.Context, f *flag.FlagSet) {
	f.StringVar(&flag.path, "options", "", "Options spec file path for VM deployment")
	f.StringVar(&flag.json, "options.json", "", "Options spec string for VM deployment")
}

func (flag *OptionsFlag) Process(ctx context.Context) error {
	if len(flag.path) != 0 && len(flag.json) != 0 {
		return errors.New("only one options spec could be specified")
	}

	var r io.Reader
	if len(flag.path) != 0 {
		f, err := os.Open(flag.path)
		if err != nil {
			return err
		}
		defer f.Close()
		r = f
	} else if len(flag.json) != 0 {
		r = strings.NewReader(flag.json)
	} else {
		return nil
	}

	if err := json.NewDecoder(r).Decode(&flag.Options); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiUrlPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiUrlPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for len(ids) > 0 {
		var tIds []uint
		if len(ids) > 100 {
			tIds = ids[:100]
			ids = ids[100:]
		} else {
			tIds = ids
			ids = nil
		}
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<commit_msg>Added CheckNumber<commit_after>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiUrlPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiUrlPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for len(ids) > 0 {
		var tIds []uint
		if len(ids) > 100 {
			tIds = ids[:100]
			ids = ids[100:]
		} else {
			tIds = ids
			ids = nil
		}
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers ...uint) (map[uint]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers...)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint]Number)
	for n, data := range ns {
		number, err := strconv.Atoi(n)
		if err != nil {
			continue
		}
		toRet[uint(number)] = data
	}
	return toRet, nil
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<|endoftext|>


<commit_before>package logger

// Simple Verbose & Debug Logger for Go
//
// (C) Sep/2014 by Marco Paganini <paganini AT paganini DOT net>

import (
	"log"
	"os"
)

// Base Logger struct using anonymous *log.Logger method
type Logger struct {
	*log.Logger
	verbose int
	debug   int
}

// Creates a new Logger instance based on log and returns it
//
// Returns:
//   logger instance
func New(prefix string) *Logger {
	o := log.New(os.Stderr, prefix, 0)
	return &Logger{o, 0, 0}
}

// Set Verbose level
func (o *Logger) SetVerboseLevel(n int) {
	o.verbose = n
}

// Set Debugging level
func (o *Logger) SetDebugLevel(n int) {
	o.debug = n
}

// PrintLn the message if the current verbose level >= 'level'
func (o *Logger) Verboseln(level int, v ...interface{}) {
	if o.verbose >= level {
		o.Println(v...)
	}
}

// Printf the message and parameters if the current verbose level >= 'level'
func (o *Logger) Verbosef(level int, format string, v ...interface{}) {
	if o.verbose >= level {
		o.Printf(format, v...)
	}
}

// PrintLn the message if the current Debug level >= 'level'
func (o *Logger) Debugln(level int, v ...interface{}) {
	o.SetFlags(log.Lshortfile)
	if o.debug >= level {
		o.Println(v...)
	}
	o.SetFlags(0)
}

// Printf the message and parameters if the current Debug level >= 'level'
func (o *Logger) Debugf(level int, format string, v ...interface{}) {
	o.SetFlags(log.Lshortfile)
	if o.debug >= level {
		o.Printf(format, v...)
	}
	o.SetFlags(0)
}
<commit_msg>Added multi-stream output support.<commit_after>// Simple log replacement for go with Verbosity and Debug levels, and
// multi-stream output support.
//
// (C) by Marco Paganini <paganini AT paganini DOT net>

package logger

import (
	"fmt"
	"io"
	"log"
	"os"
)

// Logger represents a logger object
type Logger struct {
	*log.Logger
	verbose int
	debug   int
	streams []*os.File
}

// New Creates a new Logger instance.
func New(prefix string) *Logger {
	o := log.New(os.Stderr, prefix, 0)
	return &Logger{o, 0, 0, []*os.File{os.Stdout}}
}

// SetVerboseLevel sets the verbosity level for this log instance.
func (o *Logger) SetVerboseLevel(n int) {
	o.verbose = n
}

// SetDebugLevel sets the debugging level for this log instance.
func (o *Logger) SetDebugLevel(n int) {
	o.debug = n
}

// SetOutput sets the output streams to the list of writers.
func (o *Logger) SetOutput(streams []*os.File) {
	o.streams = streams
}

// writeString sends the string to all defined outputs.
func (o *Logger) writeString(s string) {
	for _, w := range o.streams {
		io.WriteString(w, s)
	}
}

// Verboseln prints the message to the output streams, followed by a newline,
// if the current verbose level is greater than or equal the specified
// verbosity level.
func (o *Logger) Verboseln(level int, v ...interface{}) {
	if o.verbose >= level {
		o.writeString(fmt.Sprintln(v...))
	}
}

// Verbosef uses the formatting string and variables to print a message to the
// output streams if the current verbose level is greater than or equal to the
// specified verbose level.
func (o *Logger) Verbosef(level int, format string, v ...interface{}) {
	if o.verbose >= level {
		o.writeString(fmt.Sprintf(format, v...))
	}
}

// Debugln prints the message to the output streams, followed by a newline,
// if the current debugging level is greater than or equal the specified
// debugging level.
func (o *Logger) Debugln(level int, v ...interface{}) {
	o.SetFlags(log.Lshortfile)
	if o.debug >= level {
		o.writeString(fmt.Sprintln(v...))
	}
	o.SetFlags(0)
}

// Debugf uses the formatting string and variables to print a message to the
// output streams if the current debugging level is greater than or equal to the
// specified debugging level.
func (o *Logger) Debugf(level int, format string, v ...interface{}) {
	o.SetFlags(log.Lshortfile)
	if o.debug >= level {
		o.writeString(fmt.Sprintf(format, v...))
	}
	o.SetFlags(0)
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
)

const (
	LOG_LEVEL_OFF = iota
	LOG_LEVEL_FATAL
	LOG_LEVEL_ERROR
	LOG_LEVEL_WARN
	LOG_LEVEL_INFO
	LOG_LEVEL_DEBUG
	LOG_LEVEL_TRACE
	LOG_LEVEL_ALL
)

type Logger struct {
	level  int
	prefix string
}

func NewLogger(level int, prefix string) *Logger {
	return &Logger{
		level:  level,
		prefix: prefix,
	}
}

func (l *Logger) Fatal(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_FATAL {
		l.Log("[fatal]", format, v...)
	}
}

func (l *Logger) Error(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_ERROR {
		l.Log("[error]", format, v...)
	}
}

func (l *Logger) Warn(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_WARN {
		l.Log("[warn] ", format, v...)
	}
}

func (l *Logger) Info(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_INFO {
		l.Log("[info] ", format, v...)
	}
}

func (l *Logger) Debug(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_DEBUG {
		l.Log("[debug]", format, v...)
	}
}

func (l *Logger) Trace(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_TRACE {
		l.Log("[trace]", format, v...)
	}
}

func (l *Logger) Log(level string, format string, v ...interface{}) {
	out := fmt.Sprintf(fmt.Sprintf("%s\n", format), v...)

	if l.prefix != "" {
		fmt.Printf("%s (%s) %s", level, l.prefix, out)
	} else {
		fmt.Printf("%s %s", level, out)
	}
}
<commit_msg>Add timestamp<commit_after>package logger

import (
	"fmt"
	"time"
)

const (
	LOG_LEVEL_OFF = iota
	LOG_LEVEL_FATAL
	LOG_LEVEL_ERROR
	LOG_LEVEL_WARN
	LOG_LEVEL_INFO
	LOG_LEVEL_DEBUG
	LOG_LEVEL_TRACE
	LOG_LEVEL_ALL
)

type Logger struct {
	level  int
	prefix string
}

func NewLogger(level int, prefix string) *Logger {
	return &Logger{
		level:  level,
		prefix: prefix,
	}
}

func (l *Logger) Fatal(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_FATAL {
		l.Log("[fatal]", format, v...)
	}
}

func (l *Logger) Error(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_ERROR {
		l.Log("[error]", format, v...)
	}
}

func (l *Logger) Warn(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_WARN {
		l.Log("[warn] ", format, v...)
	}
}

func (l *Logger) Info(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_INFO {
		l.Log("[info] ", format, v...)
	}
}

func (l *Logger) Debug(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_DEBUG {
		l.Log("[debug]", format, v...)
	}
}

func (l *Logger) Trace(format string, v ...interface{}) {
	if l.level >= LOG_LEVEL_TRACE {
		l.Log("[trace]", format, v...)
	}
}

func (l *Logger) Log(level string, format string, v ...interface{}) {
	tim := time.Now().Format("2006-01-02 15:04:05.000000")
	out := fmt.Sprintf(fmt.Sprintf("%s\n", format), v...)

	if l.prefix != "" {
		fmt.Printf("[%s] %s (%s) %s", tim, level, l.prefix, out)
	} else {
		fmt.Printf("[%s] %s %s", tim, level, out)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"fmt"
	"github.com/daaku/go.zipexe"
	"go/build"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

func operationAppend(pkg *build.Package) {
	if runtime.GOOS == "windows" {
		fmt.Println("#### WARNING ! ####")
		fmt.Println("`rice append` is known not to work under windows because the `zip` command is not available. Please let me know if you got this to work (and how).")
	}

	// MARKED FOR DELETION
	// This is actually not required, the append command now has the option --exec required.
	// // check if package is a command
	// if !pkg.IsCommand() {
	// 	fmt.Println("Error: can not append to non-main package. Please follow instructions at github.com/GeertJohan/go.rice")
	// 	os.Exit(1)
	// }

	// find boxes for this command
	boxMap := findBoxes(pkg)

	fmt.Printf("pkg.Dir: %s\n", pkg.Dir)

	// notify user when no calls to rice.FindBox are made (is this an error and therefore os.Exit(1) ?
	if len(boxMap) == 0 {
		fmt.Println("no calls to rice.FindBox() found")
	}

	verbosef("\n")

	// create tmp zipfile
	tmpZipfileName := filepath.Join(os.TempDir(), fmt.Sprintf("ricebox-%d-%s.zip", time.Now().Unix(), randomString(10)))
	verbosef("Will create tmp zipfile: %s\n", tmpZipfileName)
	tmpZipfile, err := os.Create(tmpZipfileName)
	if err != nil {
		fmt.Printf("Error creating tmp zipfile: %s\n", err)
		os.Exit(1)
	}

	// find abs path for binary file
	binfileName, err := filepath.Abs(flags.Append.Executable)
	if err != nil {
		fmt.Printf("Error finding absolute path for executable to append: %s\n", err)
		os.Exit(1)
	}
	verbosef("Will append to file: %s\n", binfileName)

	// check that command doesn't already have zip appended
	if rd, _ := zipexe.Open(binfileName); rd != nil {
		fmt.Printf("Cannot append to already appended executable. Please remove %s and build a fresh one.\n", binfileName)
		os.Exit(1)
	}

	// open binfile
	binfile, err := os.OpenFile(binfileName, os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Printf("Error: unable to open executable file: %s\n", err)
		os.Exit(1)
	}

	// create zip.Writer
	zipWriter := zip.NewWriter(tmpZipfile)

	for boxname := range boxMap {
		appendedBoxName := strings.Replace(boxname, `/`, `-`, -1)

		// walk box path's and insert files
		boxPath := filepath.Clean(filepath.Join(pkg.Dir, boxname))
		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
			// create zipFilename
			zipFileName := filepath.Join(appendedBoxName, strings.TrimPrefix(path, boxPath))

			// write directories as empty file with comment "dir"
			if info.IsDir() {
				_, err := zipWriter.CreateHeader(&zip.FileHeader{
					Name:    zipFileName,
					Comment: "dir",
				})
				if err != nil {
					fmt.Printf("Error creating dir in tmp zip: %s\n", err)
					os.Exit(1)
				}
				return nil
			}

			// create zipFileWriter
			zipFileWriter, err := zipWriter.Create(zipFileName)
			if err != nil {
				fmt.Printf("Error creating file in tmp zip: %s\n", err)
				os.Exit(1)
			}
			srcFile, err := os.Open(path)
			if err != nil {
				fmt.Printf("Error opening file to append: %s\n", err)
				os.Exit(1)
			}
			_, err = io.Copy(zipFileWriter, srcFile)
			if err != nil {
				fmt.Printf("Error copying file contents to zip: %s\n", err)
				os.Exit(1)
			}
			srcFile.Close()

			return nil
		})
	}

	err = zipWriter.Close()
	if err != nil {
		fmt.Printf("Error closing tmp zipfile: %s\n", err)
		os.Exit(1)
	}

	err = tmpZipfile.Sync()
	if err != nil {
		fmt.Printf("Error syncing tmp zipfile: %s\n", err)
		os.Exit(1)
	}
	_, err = tmpZipfile.Seek(0, 0)
	if err != nil {
		fmt.Printf("Error seeking tmp zipfile: %s\n", err)
		os.Exit(1)
	}
	_, err = binfile.Seek(0, 2)
	if err != nil {
		fmt.Printf("Error seeking bin file: %s\n", err)
		os.Exit(1)
	}

	_, err = io.Copy(binfile, tmpZipfile)
	if err != nil {
		fmt.Printf("Error appending zipfile to executable: %s\n", err)
		os.Exit(1)
	}

	zipA := exec.Command("zip", "-A", binfileName)
	err = zipA.Run()
	if err != nil {
		fmt.Printf("Error setting zip offset: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>Remove debug logging<commit_after>package main

import (
	"archive/zip"
	"fmt"
	"github.com/daaku/go.zipexe"
	"go/build"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

func operationAppend(pkg *build.Package) {
	if runtime.GOOS == "windows" {
		fmt.Println("#### WARNING ! ####")
		fmt.Println("`rice append` is known not to work under windows because the `zip` command is not available. Please let me know if you got this to work (and how).")
	}

	// MARKED FOR DELETION
	// This is actually not required, the append command now has the option --exec required.
	// // check if package is a command
	// if !pkg.IsCommand() {
	// 	fmt.Println("Error: can not append to non-main package. Please follow instructions at github.com/GeertJohan/go.rice")
	// 	os.Exit(1)
	// }

	// find boxes for this command
	boxMap := findBoxes(pkg)

	// notify user when no calls to rice.FindBox are made (is this an error and therefore os.Exit(1) ?
	if len(boxMap) == 0 {
		fmt.Println("no calls to rice.FindBox() found")
	}

	verbosef("\n")

	// create tmp zipfile
	tmpZipfileName := filepath.Join(os.TempDir(), fmt.Sprintf("ricebox-%d-%s.zip", time.Now().Unix(), randomString(10)))
	verbosef("Will create tmp zipfile: %s\n", tmpZipfileName)
	tmpZipfile, err := os.Create(tmpZipfileName)
	if err != nil {
		fmt.Printf("Error creating tmp zipfile: %s\n", err)
		os.Exit(1)
	}

	// find abs path for binary file
	binfileName, err := filepath.Abs(flags.Append.Executable)
	if err != nil {
		fmt.Printf("Error finding absolute path for executable to append: %s\n", err)
		os.Exit(1)
	}
	verbosef("Will append to file: %s\n", binfileName)

	// check that command doesn't already have zip appended
	if rd, _ := zipexe.Open(binfileName); rd != nil {
		fmt.Printf("Cannot append to already appended executable. Please remove %s and build a fresh one.\n", binfileName)
		os.Exit(1)
	}

	// open binfile
	binfile, err := os.OpenFile(binfileName, os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Printf("Error: unable to open executable file: %s\n", err)
		os.Exit(1)
	}

	// create zip.Writer
	zipWriter := zip.NewWriter(tmpZipfile)

	for boxname := range boxMap {
		appendedBoxName := strings.Replace(boxname, `/`, `-`, -1)

		// walk box path's and insert files
		boxPath := filepath.Clean(filepath.Join(pkg.Dir, boxname))
		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
			// create zipFilename
			zipFileName := filepath.Join(appendedBoxName, strings.TrimPrefix(path, boxPath))

			// write directories as empty file with comment "dir"
			if info.IsDir() {
				_, err := zipWriter.CreateHeader(&zip.FileHeader{
					Name:    zipFileName,
					Comment: "dir",
				})
				if err != nil {
					fmt.Printf("Error creating dir in tmp zip: %s\n", err)
					os.Exit(1)
				}
				return nil
			}

			// create zipFileWriter
			zipFileWriter, err := zipWriter.Create(zipFileName)
			if err != nil {
				fmt.Printf("Error creating file in tmp zip: %s\n", err)
				os.Exit(1)
			}
			srcFile, err := os.Open(path)
			if err != nil {
				fmt.Printf("Error opening file to append: %s\n", err)
				os.Exit(1)
			}
			_, err = io.Copy(zipFileWriter, srcFile)
			if err != nil {
				fmt.Printf("Error copying file contents to zip: %s\n", err)
				os.Exit(1)
			}
			srcFile.Close()

			return nil
		})
	}

	err = zipWriter.Close()
	if err != nil {
		fmt.Printf("Error closing tmp zipfile: %s\n", err)
		os.Exit(1)
	}

	err = tmpZipfile.Sync()
	if err != nil {
		fmt.Printf("Error syncing tmp zipfile: %s\n", err)
		os.Exit(1)
	}
	_, err = tmpZipfile.Seek(0, 0)
	if err != nil {
		fmt.Printf("Error seeking tmp zipfile: %s\n", err)
		os.Exit(1)
	}
	_, err = binfile.Seek(0, 2)
	if err != nil {
		fmt.Printf("Error seeking bin file: %s\n", err)
		os.Exit(1)
	}

	_, err = io.Copy(binfile, tmpZipfile)
	if err != nil {
		fmt.Printf("Error appending zipfile to executable: %s\n", err)
		os.Exit(1)
	}

	zipA := exec.Command("zip", "-A", binfileName)
	err = zipA.Run()
	if err != nil {
		fmt.Printf("Error setting zip offset: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package generator

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	loggingconfig "github.com/rancher/rancher/pkg/controllers/user/logging/config"
	"github.com/rancher/rancher/pkg/project"
	mgmtv3 "github.com/rancher/types/apis/management.cattle.io/v3"

	"github.com/pkg/errors"
	k8scorev1 "k8s.io/api/core/v1"
)

var tmplCache = template.New("template")

func init() {
	tmplCache = tmplCache.Funcs(template.FuncMap{"escapeString": escapeString})
	tmplCache = template.Must(tmplCache.Parse(SourceTemplate))
	tmplCache = template.Must(tmplCache.Parse(FilterTemplate))
	tmplCache = template.Must(tmplCache.Parse(MatchTemplate))
	tmplCache = template.Must(tmplCache.Parse(Template))
}

func GenerateConfig(tempalteName string, conf interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	if err := tmplCache.ExecuteTemplate(buf, tempalteName, conf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func GenerateClusterConfig(logging mgmtv3.ClusterLoggingSpec, excludeNamespaces, certDir string) ([]byte, error) {
	wl, err := newWrapClusterLogging(logging, excludeNamespaces, certDir)
	if err != nil {
		return nil, errors.Wrap(err, "to wraper cluster logging failed")
	}

	if wl == nil {
		return []byte{}, nil
	}

	if logging.SyslogConfig != nil && logging.SyslogConfig.Token != "" {
		if err = ValidateSyslogToken(wl); err != nil {
			return nil, err
		}
	}

	if err = ValidateCustomTags(wl); err != nil {
		return nil, err
	}

	validateData := *wl
	if logging.FluentForwarderConfig != nil && wl.EnableShareKey {
		validateData.EnableShareKey = false //skip generate precan configure included ruby code
	}
	if err = ValidateCustomTarget(validateData); err != nil {
		return nil, err
	}

	buf, err := GenerateConfig("cluster-template", wl)
	if err != nil {
		return nil, errors.Wrap(err, "generate cluster config file failed")
	}

	return buf, nil
}

func GenerateProjectConfig(projectLoggings []*mgmtv3.ProjectLogging, namespaces []*k8scorev1.Namespace, systemProjectID, certDir string) ([]byte, error) {
	var wl []ProjectLoggingTemplateWrap
	for _, v := range projectLoggings {
		var grepNamespace []string
		for _, v2 := range namespaces {
			if nsProjectName, ok := v2.Annotations[project.ProjectIDAnn]; ok && nsProjectName == v.Spec.ProjectName {
				namespacePattern := loggingconfig.GetNamespacePattern(v2.Name)
				grepNamespace = append(grepNamespace, namespacePattern)
			}
		}

		if len(grepNamespace) == 0 {
			continue
		}

		formatgrepNamespace := fmt.Sprintf("(%s)", strings.Join(grepNamespace, "|"))
		isSystemProject := v.Spec.ProjectName == systemProjectID
		wpl, err := newWrapProjectLogging(v.Spec, formatgrepNamespace, certDir, isSystemProject)
		if err != nil {
			return nil, err
		}

		if wpl == nil {
			continue
		}

		if wpl.SyslogConfig.Token != "" {
			if err = ValidateSyslogToken(wpl); err != nil {
				return nil, err
			}
		}

		if err = ValidateCustomTags(wpl); err != nil {
			return nil, err
		}

		validateData := *wpl
		if v.Spec.FluentForwarderConfig != nil && wpl.EnableShareKey {
			validateData.EnableShareKey = false //skip generate precan configure included ruby code
		}
		if err = ValidateCustomTarget(validateData); err != nil {
			return nil, err
		}

		wl = append(wl, *wpl)
	}

	buf, err := GenerateConfig("project-template", wl)
	if err != nil {
		return nil, errors.Wrap(err, "generate project config file failed")
	}

	return buf, nil
}
<commit_msg>sort grep namespace list for project logging<commit_after>package generator

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"text/template"

	loggingconfig "github.com/rancher/rancher/pkg/controllers/user/logging/config"
	"github.com/rancher/rancher/pkg/project"
	mgmtv3 "github.com/rancher/types/apis/management.cattle.io/v3"

	"github.com/pkg/errors"
	k8scorev1 "k8s.io/api/core/v1"
)

var tmplCache = template.New("template")

func init() {
	tmplCache = tmplCache.Funcs(template.FuncMap{"escapeString": escapeString})
	tmplCache = template.Must(tmplCache.Parse(SourceTemplate))
	tmplCache = template.Must(tmplCache.Parse(FilterTemplate))
	tmplCache = template.Must(tmplCache.Parse(MatchTemplate))
	tmplCache = template.Must(tmplCache.Parse(Template))
}

func GenerateConfig(tempalteName string, conf interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	if err := tmplCache.ExecuteTemplate(buf, tempalteName, conf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func GenerateClusterConfig(logging mgmtv3.ClusterLoggingSpec, excludeNamespaces, certDir string) ([]byte, error) {
	wl, err := newWrapClusterLogging(logging, excludeNamespaces, certDir)
	if err != nil {
		return nil, errors.Wrap(err, "to wraper cluster logging failed")
	}

	if wl == nil {
		return []byte{}, nil
	}

	if logging.SyslogConfig != nil && logging.SyslogConfig.Token != "" {
		if err = ValidateSyslogToken(wl); err != nil {
			return nil, err
		}
	}

	if err = ValidateCustomTags(wl); err != nil {
		return nil, err
	}

	validateData := *wl
	if logging.FluentForwarderConfig != nil && wl.EnableShareKey {
		validateData.EnableShareKey = false //skip generate precan configure included ruby code
	}
	if err = ValidateCustomTarget(validateData); err != nil {
		return nil, err
	}

	buf, err := GenerateConfig("cluster-template", wl)
	if err != nil {
		return nil, errors.Wrap(err, "generate cluster config file failed")
	}

	return buf, nil
}

func GenerateProjectConfig(projectLoggings []*mgmtv3.ProjectLogging, namespaces []*k8scorev1.Namespace, systemProjectID, certDir string) ([]byte, error) {
	var wl []ProjectLoggingTemplateWrap
	for _, v := range projectLoggings {
		var grepNamespace []string
		for _, v2 := range namespaces {
			if nsProjectName, ok := v2.Annotations[project.ProjectIDAnn]; ok && nsProjectName == v.Spec.ProjectName {
				namespacePattern := loggingconfig.GetNamespacePattern(v2.Name)
				grepNamespace = append(grepNamespace, namespacePattern)
			}
		}

		if len(grepNamespace) == 0 {
			continue
		}

		sort.Strings(grepNamespace)
		formatgrepNamespace := fmt.Sprintf("(%s)", strings.Join(grepNamespace, "|"))
		isSystemProject := v.Spec.ProjectName == systemProjectID
		wpl, err := newWrapProjectLogging(v.Spec, formatgrepNamespace, certDir, isSystemProject)
		if err != nil {
			return nil, err
		}

		if wpl == nil {
			continue
		}

		if wpl.SyslogConfig.Token != "" {
			if err = ValidateSyslogToken(wpl); err != nil {
				return nil, err
			}
		}

		if err = ValidateCustomTags(wpl); err != nil {
			return nil, err
		}

		validateData := *wpl
		if v.Spec.FluentForwarderConfig != nil && wpl.EnableShareKey {
			validateData.EnableShareKey = false //skip generate precan configure included ruby code
		}
		if err = ValidateCustomTarget(validateData); err != nil {
			return nil, err
		}

		wl = append(wl, *wpl)
	}

	buf, err := GenerateConfig("project-template", wl)
	if err != nil {
		return nil, errors.Wrap(err, "generate project config file failed")
	}

	return buf, nil
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"time"
)

type msgType uint8

// DebugMode Enable debug mode
var DebugMode bool

const (
	messageConst  msgType = 0
	message2Const msgType = 1
	warningConst  msgType = 2
	debugConst    msgType = 3
	errorConst    msgType = 0x80
)

// Fatal show message with line break at the end and exit to OS.
func Fatal(msg ...interface{}) {
	pln(errorConst, msg...)
	os.Exit(-1)
}

// Errorln message with line break at the end.
func Errorln(msg ...interface{}) {
	pln(errorConst, msg...)
}

// Println shows message on screen with line break at the end.
func Println(msg ...interface{}) {
	pln(messageConst, msg...)
}

// Debugln shows debug message on screen with line break at the end.
// If debug mode is not active no message is displayed
func Debugln(msg ...interface{}) {
	pln(debugConst, msg...)
}

func pln(m msgType, msg ...interface{}) {
	if m == debugConst && !DebugMode {
		return
	}

	var debugInfo string
	var color string
	var prefix string

	switch m {
	case messageConst:
		color = "\x1b[37m" // White
		prefix = "msg"
	case message2Const:
		color = "\x1b[92m" // Light green
		prefix = "msg"
	case warningConst:
		color = "\x1b[93m" // Light Yellow
		prefix = "warning"
	case debugConst:
		color = "\x1b[96m" // Light Cyan
		prefix = "debug"
	case errorConst:
		color = "\x1b[91m" // Light Red
		prefix = "error"
	}

	if DebugMode {
		_, fn, line, _ := runtime.Caller(2)
		fn = filepath.Base(fn)
		debugInfo = fmt.Sprintf("%s:%d ", fn, line)
	}

	fmt.Printf("%s%s [%s] %s%s\033[0;00m\n",
		color,
		time.Now().UTC().Format("2006/01/02 15:04:05"),
		prefix,
		debugInfo,
		fmt.Sprint(msg...))
}
<commit_msg>implement lists as array<commit_after>package log

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"time"
)

type msgType uint8

const (
	MessageLog  msgType = 0
	Message2Log msgType = 1
	WarningLog  msgType = 2
	DebugLog    msgType = 3
	ErrorLog    msgType = 4

	ColorReset = "\033[0;00m" // Return terminal to default color
)

var (

	// DebugMode Enable debug mode
	DebugMode bool

	Colors = []string{
		MessageLog:  "\x1b[37m", // White
		Message2Log: "\x1b[92m", // Light green
		WarningLog:  "\x1b[93m", // Light Yellow
		DebugLog:    "\x1b[96m", // Light Cyan
		ErrorLog:    "\x1b[91m", // Light Red
	}

	Prefixes = []string{
		MessageLog:  "msg",
		Message2Log: "msg",
		WarningLog:  "warning",
		DebugLog:    "debug",
		ErrorLog:    "error",
	}
)

// Fatal show message with line break at the end and exit to OS.
func Fatal(msg ...interface{}) {
	pln(ErrorLog, msg...)
	os.Exit(-1)
}

// Errorln message with line break at the end.
func Errorln(msg ...interface{}) {
	pln(ErrorLog, msg...)
}

// Println shows message on screen with line break at the end.
func Println(msg ...interface{}) {
	pln(MessageLog, msg...)
}

// Debugln shows debug message on screen with line break at the end.
// If debug mode is not active no message is displayed
func Debugln(msg ...interface{}) {
	pln(DebugLog, msg...)
}

func pln(m msgType, msg ...interface{}) {
	if m == DebugLog && !DebugMode {
		return
	}

	var debugInfo string

	if DebugMode {
		_, fn, line, _ := runtime.Caller(2)
		fn = filepath.Base(fn)
		debugInfo = fmt.Sprintf("%s:%d ", fn, line)
	}

	fmt.Printf("%s%s [%s] %s%s\033[0;00m\n",
		Colors[m],
		time.Now().UTC().Format("2006/01/02 15:04:05"),
		Prefixes[m],
		debugInfo,
		fmt.Sprint(msg...))
}
<|endoftext|>


<commit_before>package filer2

import (
	"sort"
	"log"
	"math"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
	for _, c := range chunks {
		t := uint64(c.Offset + int64(c.Size))
		if size < t {
			size = t
		}
	}
	return
}

func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {

	visibles := nonOverlappingVisibleIntervals(chunks)

	fileIds := make(map[string]bool)
	for _, interval := range visibles {
		fileIds[interval.fileId] = true
	}
	for _, chunk := range chunks {
		if found := fileIds[chunk.FileId]; found {
			compacted = append(compacted, chunk)
		} else {
			garbage = append(garbage, chunk)
		}
	}

	return
}

func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {

	fileIds := make(map[string]bool)
	for _, interval := range newChunks {
		fileIds[interval.FileId] = true
	}
	for _, chunk := range oldChunks {
		if found := fileIds[chunk.FileId]; !found {
			unused = append(unused, chunk)
		}
	}

	return
}

func logPrintf(name string, visibles []*visibleInterval) {

	// return

	log.Printf("%s len %d", name, len(visibles))
	for _, v := range visibles {
		log.Printf("%s:  => %+v", name, v)
	}
}

func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) {

	sort.Slice(chunks, func(i, j int) bool {
		if chunks[i].Offset < chunks[j].Offset {
			return true
		}
		if chunks[i].Offset == chunks[j].Offset {
			return chunks[i].Mtime < chunks[j].Mtime
		}
		return false
	})

	if len(chunks) == 0 {
		return
	}

	var parallelIntervals, intervals []*visibleInterval
	var minStopInterval, upToDateInterval *visibleInterval
	watermarkStart := chunks[0].Offset
	for _, chunk := range chunks {
		log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
		logPrintf("parallelIntervals", parallelIntervals)
		for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset {
			logPrintf("parallelIntervals loop 1", parallelIntervals)
			logPrintf("parallelIntervals loop 1 intervals", intervals)
			minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
			nextStop := min(minStopInterval.stop, chunk.Offset)
			intervals = append(intervals, newVisibleInterval(
				max(watermarkStart, minStopInterval.start),
				nextStop,
				upToDateInterval.fileId,
				upToDateInterval.modifiedTime,
			))
			watermarkStart = nextStop
			logPrintf("parallelIntervals loop intervals =>", intervals)

			// remove processed intervals, possibly multiple
			var remaining []*visibleInterval
			for _, interval := range parallelIntervals {
				if interval.stop != watermarkStart {
					remaining = append(remaining, interval)
				}
			}
			parallelIntervals = remaining
			logPrintf("parallelIntervals loop 2", parallelIntervals)
			logPrintf("parallelIntervals loop 2 intervals", intervals)
		}
		parallelIntervals = append(parallelIntervals, newVisibleInterval(
			chunk.Offset,
			chunk.Offset+int64(chunk.Size),
			chunk.FileId,
			chunk.Mtime,
		))
	}

	logPrintf("parallelIntervals loop 3", parallelIntervals)
	logPrintf("parallelIntervals loop 3 intervals", intervals)
	for len(parallelIntervals) > 0 {
		minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
		intervals = append(intervals, newVisibleInterval(
			max(watermarkStart, minStopInterval.start),
			minStopInterval.stop,
			upToDateInterval.fileId,
			upToDateInterval.modifiedTime,
		))
		watermarkStart = minStopInterval.stop

		// remove processed intervals, possibly multiple
		var remaining []*visibleInterval
		for _, interval := range parallelIntervals {
			if interval.stop != watermarkStart {
				remaining = append(remaining, interval)
			}
		}
		parallelIntervals = remaining
	}
	logPrintf("parallelIntervals loop 4", parallelIntervals)
	logPrintf("intervals", intervals)

	// merge connected intervals, now the intervals are non-intersecting
	var lastIntervalIndex int
	var prevIntervalIndex int
	for i, interval := range intervals {
		if i == 0 {
			prevIntervalIndex = i
			lastIntervalIndex = i
			continue
		}
		if intervals[i-1].fileId != interval.fileId ||
			intervals[i-1].stop < intervals[i].start {
			visibles = append(visibles, newVisibleInterval(
				intervals[prevIntervalIndex].start,
				intervals[i-1].stop,
				intervals[prevIntervalIndex].fileId,
				intervals[prevIntervalIndex].modifiedTime,
			))
			prevIntervalIndex = i
		}
		lastIntervalIndex = i
		logPrintf("intervals loop 1 visibles", visibles)
	}

	visibles = append(visibles, newVisibleInterval(
		intervals[prevIntervalIndex].start,
		intervals[lastIntervalIndex].stop,
		intervals[prevIntervalIndex].fileId,
		intervals[prevIntervalIndex].modifiedTime,
	))

	logPrintf("visibles", visibles)

	return
}

func findMinStopInterval(intervals []*visibleInterval) (minStopInterval, upToDateInterval *visibleInterval) {
	var latestMtime int64
	latestIntervalIndex := 0
	minStop := int64(math.MaxInt64)
	minIntervalIndex := 0
	for i, interval := range intervals {
		if minStop > interval.stop {
			minIntervalIndex = i
			minStop = interval.stop
		}
		if latestMtime < interval.modifiedTime {
			latestMtime = interval.modifiedTime
			latestIntervalIndex = i
		}
	}
	minStopInterval = intervals[minIntervalIndex]
	upToDateInterval = intervals[latestIntervalIndex]
	return
}

// find non-overlapping visible intervals
// visible interval map to one file chunk

type visibleInterval struct {
	start        int64
	stop         int64
	modifiedTime int64
	fileId       string
}

func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *visibleInterval {
	return &visibleInterval{start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime}
}

func min(x, y int64) int64 {
	if x <= y {
		return x
	}
	return y
}

func max(x, y int64) int64 {
	if x > y {
		return x
	}
	return y
}
<commit_msg>reduce logs<commit_after>package filer2

import (
	"sort"
	"log"
	"math"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
	for _, c := range chunks {
		t := uint64(c.Offset + int64(c.Size))
		if size < t {
			size = t
		}
	}
	return
}

func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {

	visibles := nonOverlappingVisibleIntervals(chunks)

	fileIds := make(map[string]bool)
	for _, interval := range visibles {
		fileIds[interval.fileId] = true
	}
	for _, chunk := range chunks {
		if found := fileIds[chunk.FileId]; found {
			compacted = append(compacted, chunk)
		} else {
			garbage = append(garbage, chunk)
		}
	}

	return
}

func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {

	fileIds := make(map[string]bool)
	for _, interval := range newChunks {
		fileIds[interval.FileId] = true
	}
	for _, chunk := range oldChunks {
		if found := fileIds[chunk.FileId]; !found {
			unused = append(unused, chunk)
		}
	}

	return
}

func logPrintf(name string, visibles []*visibleInterval) {

	return

	log.Printf("%s len %d", name, len(visibles))
	for _, v := range visibles {
		log.Printf("%s:  => %+v", name, v)
	}
}

func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) {

	sort.Slice(chunks, func(i, j int) bool {
		if chunks[i].Offset < chunks[j].Offset {
			return true
		}
		if chunks[i].Offset == chunks[j].Offset {
			return chunks[i].Mtime < chunks[j].Mtime
		}
		return false
	})

	if len(chunks) == 0 {
		return
	}

	var parallelIntervals, intervals []*visibleInterval
	var minStopInterval, upToDateInterval *visibleInterval
	watermarkStart := chunks[0].Offset
	for _, chunk := range chunks {
		// log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
		logPrintf("parallelIntervals", parallelIntervals)
		for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset {
			logPrintf("parallelIntervals loop 1", parallelIntervals)
			logPrintf("parallelIntervals loop 1 intervals", intervals)
			minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
			nextStop := min(minStopInterval.stop, chunk.Offset)
			intervals = append(intervals, newVisibleInterval(
				max(watermarkStart, minStopInterval.start),
				nextStop,
				upToDateInterval.fileId,
				upToDateInterval.modifiedTime,
			))
			watermarkStart = nextStop
			logPrintf("parallelIntervals loop intervals =>", intervals)

			// remove processed intervals, possibly multiple
			var remaining []*visibleInterval
			for _, interval := range parallelIntervals {
				if interval.stop != watermarkStart {
					remaining = append(remaining, interval)
				}
			}
			parallelIntervals = remaining
			logPrintf("parallelIntervals loop 2", parallelIntervals)
			logPrintf("parallelIntervals loop 2 intervals", intervals)
		}
		parallelIntervals = append(parallelIntervals, newVisibleInterval(
			chunk.Offset,
			chunk.Offset+int64(chunk.Size),
			chunk.FileId,
			chunk.Mtime,
		))
	}

	logPrintf("parallelIntervals loop 3", parallelIntervals)
	logPrintf("parallelIntervals loop 3 intervals", intervals)
	for len(parallelIntervals) > 0 {
		minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
		intervals = append(intervals, newVisibleInterval(
			max(watermarkStart, minStopInterval.start),
			minStopInterval.stop,
			upToDateInterval.fileId,
			upToDateInterval.modifiedTime,
		))
		watermarkStart = minStopInterval.stop

		// remove processed intervals, possibly multiple
		var remaining []*visibleInterval
		for _, interval := range parallelIntervals {
			if interval.stop != watermarkStart {
				remaining = append(remaining, interval)
			}
		}
		parallelIntervals = remaining
	}
	logPrintf("parallelIntervals loop 4", parallelIntervals)
	logPrintf("intervals", intervals)

	// merge connected intervals, now the intervals are non-intersecting
	var lastIntervalIndex int
	var prevIntervalIndex int
	for i, interval := range intervals {
		if i == 0 {
			prevIntervalIndex = i
			lastIntervalIndex = i
			continue
		}
		if intervals[i-1].fileId != interval.fileId ||
			intervals[i-1].stop < intervals[i].start {
			visibles = append(visibles, newVisibleInterval(
				intervals[prevIntervalIndex].start,
				intervals[i-1].stop,
				intervals[prevIntervalIndex].fileId,
				intervals[prevIntervalIndex].modifiedTime,
			))
			prevIntervalIndex = i
		}
		lastIntervalIndex = i
		logPrintf("intervals loop 1 visibles", visibles)
	}

	visibles = append(visibles, newVisibleInterval(
		intervals[prevIntervalIndex].start,
		intervals[lastIntervalIndex].stop,
		intervals[prevIntervalIndex].fileId,
		intervals[prevIntervalIndex].modifiedTime,
	))

	logPrintf("visibles", visibles)

	return
}

func findMinStopInterval(intervals []*visibleInterval) (minStopInterval, upToDateInterval *visibleInterval) {
	var latestMtime int64
	latestIntervalIndex := 0
	minStop := int64(math.MaxInt64)
	minIntervalIndex := 0
	for i, interval := range intervals {
		if minStop > interval.stop {
			minIntervalIndex = i
			minStop = interval.stop
		}
		if latestMtime < interval.modifiedTime {
			latestMtime = interval.modifiedTime
			latestIntervalIndex = i
		}
	}
	minStopInterval = intervals[minIntervalIndex]
	upToDateInterval = intervals[latestIntervalIndex]
	return
}

// find non-overlapping visible intervals
// visible interval map to one file chunk

type visibleInterval struct {
	start        int64
	stop         int64
	modifiedTime int64
	fileId       string
}

func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *visibleInterval {
	return &visibleInterval{start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime}
}

func min(x, y int64) int64 {
	if x <= y {
		return x
	}
	return y
}

func max(x, y int64) int64 {
	if x > y {
		return x
	}
	return y
}
<|endoftext|>


<commit_before>package mount

import (
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
	"os"
	"syscall"
	"time"
)

func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
	if input.NodeId == 1 {
		wfs.setRootAttr(out)
		return fuse.OK
	}

	_, _, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status != fuse.OK {
		return status
	}
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)

	return fuse.OK
}

func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {

	path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status != fuse.OK {
		return status
	}

	if size, ok := input.GetSize(); ok {
		glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks))
		if size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(size) {
					// this chunk is truncated
					int64Size = int64(size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			// set the new chunks and reset entry cache
			entry.Chunks = chunks
			if fh != nil {
				fh.entryViewCache = nil
			}
		}
		entry.Attributes.Mtime = time.Now().Unix()
		entry.Attributes.FileSize = size

	}

	if mode, ok := input.GetMode(); ok {
		glog.V(4).Infof("setAttr mode %o", mode)
		entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
	}

	if uid, ok := input.GetUID(); ok {
		entry.Attributes.Uid = uid
	}

	if gid, ok := input.GetGID(); ok {
		entry.Attributes.Gid = gid
	}

	if mtime, ok := input.GetMTime(); ok {
		entry.Attributes.Mtime = mtime.Unix()
	}

	if atime, ok := input.GetATime(); ok {
		entry.Attributes.Mtime = atime.Unix()
	}

	entry.Attributes.Mtime = time.Now().Unix()
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)

	if fh != nil {
		fh.dirtyMetadata = true
		return fuse.OK
	}

	return wfs.saveEntry(path, entry)

}

func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
	now := uint64(time.Now().Unix())
	out.AttrValid = 119
	out.Ino = 1
	setBlksize(&out.Attr, blockSize)
	out.Uid = wfs.option.MountUid
	out.Gid = wfs.option.MountGid
	out.Mtime = now
	out.Ctime = now
	out.Atime = now
	out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
	out.Nlink = 1
}

func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) {
	out.Ino = inode
	out.Size = filer.FileSize(entry)
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Mtime = uint64(entry.Attributes.Mtime)
	out.Ctime = uint64(entry.Attributes.Mtime)
	out.Atime = uint64(entry.Attributes.Mtime)
	out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attributes.Uid
	out.Gid = entry.Attributes.Gid
}

func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
	out.Ino = inode
	out.Size = entry.FileSize
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Atime = uint64(entry.Attr.Mtime.Unix())
	out.Mtime = uint64(entry.Attr.Mtime.Unix())
	out.Ctime = uint64(entry.Attr.Mtime.Unix())
	out.Mode = toSyscallMode(entry.Attr.Mode)
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attr.Uid
	out.Gid = entry.Attr.Gid
}

func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, inode, entry)
}

func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
}

func chmod(existing uint32, mode uint32) uint32 {
	return existing&^07777 | mode&07777
}

func toSyscallMode(mode os.FileMode) uint32 {
	return toSyscallType(mode) | uint32(mode)
}

func toSyscallType(mode os.FileMode) uint32 {
	switch mode & os.ModeType {
	case os.ModeDir:
		return syscall.S_IFDIR
	case os.ModeSymlink:
		return syscall.S_IFLNK
	case os.ModeNamedPipe:
		return syscall.S_IFIFO
	case os.ModeSocket:
		return syscall.S_IFSOCK
	case os.ModeDevice:
		return syscall.S_IFBLK
	case os.ModeCharDevice:
		return syscall.S_IFCHR
	default:
		return syscall.S_IFREG
	}
}

func toOsFileType(mode uint32) os.FileMode {
	switch mode & (syscall.S_IFMT & 0xffff) {
	case syscall.S_IFDIR:
		return os.ModeDir
	case syscall.S_IFLNK:
		return os.ModeSymlink
	case syscall.S_IFIFO:
		return os.ModeNamedPipe
	case syscall.S_IFSOCK:
		return os.ModeSocket
	case syscall.S_IFBLK:
		return os.ModeDevice
	case syscall.S_IFCHR:
		return os.ModeCharDevice
	default:
		return 0
	}
}

func toOsFileMode(mode uint32) os.FileMode {
	return toOsFileType(mode) | os.FileMode(mode&07777)
}
<commit_msg>mount2: POSIX deleted opened file nlink should be 0<commit_after>package mount

import (
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
	"os"
	"syscall"
	"time"
)

func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
	if input.NodeId == 1 {
		wfs.setRootAttr(out)
		return fuse.OK
	}

	_, _, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status == fuse.OK {
		out.AttrValid = 1
		wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
		return status
	} else {
		if fh, found := wfs.fhmap.FindFileHandle(input.NodeId); found {
			out.AttrValid = 1
			wfs.setAttrByPbEntry(&out.Attr, input.NodeId, fh.entry)
			out.Nlink = 0
			return fuse.OK
		}
	}

	return status
}

func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {

	path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status != fuse.OK {
		return status
	}

	if size, ok := input.GetSize(); ok {
		glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks))
		if size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(size) {
					// this chunk is truncated
					int64Size = int64(size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			// set the new chunks and reset entry cache
			entry.Chunks = chunks
			if fh != nil {
				fh.entryViewCache = nil
			}
		}
		entry.Attributes.Mtime = time.Now().Unix()
		entry.Attributes.FileSize = size

	}

	if mode, ok := input.GetMode(); ok {
		// glog.V(4).Infof("setAttr mode %o", mode)
		entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
	}

	if uid, ok := input.GetUID(); ok {
		entry.Attributes.Uid = uid
	}

	if gid, ok := input.GetGID(); ok {
		entry.Attributes.Gid = gid
	}

	if mtime, ok := input.GetMTime(); ok {
		entry.Attributes.Mtime = mtime.Unix()
	}

	if atime, ok := input.GetATime(); ok {
		entry.Attributes.Mtime = atime.Unix()
	}

	entry.Attributes.Mtime = time.Now().Unix()
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)

	if fh != nil {
		fh.dirtyMetadata = true
		return fuse.OK
	}

	return wfs.saveEntry(path, entry)

}

func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
	now := uint64(time.Now().Unix())
	out.AttrValid = 119
	out.Ino = 1
	setBlksize(&out.Attr, blockSize)
	out.Uid = wfs.option.MountUid
	out.Gid = wfs.option.MountGid
	out.Mtime = now
	out.Ctime = now
	out.Atime = now
	out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
	out.Nlink = 1
}

func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) {
	out.Ino = inode
	out.Size = filer.FileSize(entry)
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Mtime = uint64(entry.Attributes.Mtime)
	out.Ctime = uint64(entry.Attributes.Mtime)
	out.Atime = uint64(entry.Attributes.Mtime)
	out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attributes.Uid
	out.Gid = entry.Attributes.Gid
}

func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
	out.Ino = inode
	out.Size = entry.FileSize
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Atime = uint64(entry.Attr.Mtime.Unix())
	out.Mtime = uint64(entry.Attr.Mtime.Unix())
	out.Ctime = uint64(entry.Attr.Mtime.Unix())
	out.Mode = toSyscallMode(entry.Attr.Mode)
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attr.Uid
	out.Gid = entry.Attr.Gid
}

func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, inode, entry)
}

func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
}

func chmod(existing uint32, mode uint32) uint32 {
	return existing&^07777 | mode&07777
}

func toSyscallMode(mode os.FileMode) uint32 {
	return toSyscallType(mode) | uint32(mode)
}

func toSyscallType(mode os.FileMode) uint32 {
	switch mode & os.ModeType {
	case os.ModeDir:
		return syscall.S_IFDIR
	case os.ModeSymlink:
		return syscall.S_IFLNK
	case os.ModeNamedPipe:
		return syscall.S_IFIFO
	case os.ModeSocket:
		return syscall.S_IFSOCK
	case os.ModeDevice:
		return syscall.S_IFBLK
	case os.ModeCharDevice:
		return syscall.S_IFCHR
	default:
		return syscall.S_IFREG
	}
}

func toOsFileType(mode uint32) os.FileMode {
	switch mode & (syscall.S_IFMT & 0xffff) {
	case syscall.S_IFDIR:
		return os.ModeDir
	case syscall.S_IFLNK:
		return os.ModeSymlink
	case syscall.S_IFIFO:
		return os.ModeNamedPipe
	case syscall.S_IFSOCK:
		return os.ModeSocket
	case syscall.S_IFBLK:
		return os.ModeDevice
	case syscall.S_IFCHR:
		return os.ModeCharDevice
	default:
		return 0
	}
}

func toOsFileMode(mode uint32) os.FileMode {
	return toOsFileType(mode) | os.FileMode(mode&07777)
}
<|endoftext|>


<commit_before>package tree

/*
二叉查找树（Binary Search Tree）是指一棵空树或者具有下列性质的二叉树：
	1. 若任意节点的左子树不空，则左子树上所有节点的值均小于它的根节点的值；
	2. 若任意节点的右子树不空，则右子树上所有节点的值均大于它的根节点的值；
	3. 任意节点的左、右子树也分别为二叉查找树；
	4. 没有键值相等的节点。
*/
type BST struct {
	root *TreeNode
}

// BST Construtor
func NewBST() *BST {
	return &BST{}
}

func insertBST(root *TreeNode, el int) bool {
	if root.Val == el {
		return false
	} else if root.Val > el {
		if root.Left == nil {
			root.Left = &TreeNode{Val: el}
		} else {
			return insertBST(root.Left, el)
		}
	} else {
		if root.Right == nil {
			root.Right = &TreeNode{Val: el}
		} else {
			return insertBST(root.Right, el)
		}
	}
	return true
}

// 在二叉搜索树插入节点
func (b *BST) Insert(el int) bool {
	if b.root == nil {
		b.root = &TreeNode{Val: el}
		return true
	}
	return insertBST(b.root, el)
}

func searchBST(root *TreeNode, el int) bool {
	if root == nil {
		return false
	}
	if root.Val == el {
		return true
	} else if root.Val > el {
		return searchBST(root.Left, el)
	}
	return searchBST(root.Right, el)
}

// 在二叉搜索树查找一个节点
func (b *BST) Search(el int) bool {
	if b.root == nil {
		return false
	}
	return searchBST(b.root, el)
}

func deleteBST(node, parent *TreeNode, isLeft bool, el int) bool {
	switch {
	case node.Val == el:
		if node.Left != nil && node.Right != nil {
			// 左右子树均不空
			q, tmp := node, node.Left
			for tmp.Right != nil {
				q, tmp = tmp, tmp.Right
			}
			node.Val = tmp.Val
			if q.Val != node.Val {
				q.Right = tmp.Left
			} else {
				q.Left = tmp.Left
			}
			return true
		}

		var tmpRes *TreeNode
		if node.Left == nil && node.Right == nil {
			// 该节点为叶子节点，直接删除
			tmpRes = nil
		} else if node.Left == nil {
			// 左子树空只需重接它的右子树
			tmpRes = node.Right
		} else if node.Right == nil {
			// 右子树空则只需重接它的左子树
			tmpRes = node.Left
		}

		if isLeft {
			parent.Left = tmpRes
		} else {
			parent.Right = tmpRes
		}
		return true
	case node.Val > el:
		if node.Left != nil {
			return deleteBST(node.Left, node, true, el)
		}
	case node.Val < el:
		if node.Right != nil {
			return deleteBST(node.Right, node, false, el)
		}
	}
	return false
}

// 在二叉查找树删除结点
func (b *BST) Delete(el int) bool {
	if b.root == nil {
		return false
	}
	if b.root.Val == el {
		tmp := &TreeNode{Val: 0}
		tmp.Left = b.root
		res := deleteBST(b.root, tmp, true, el)
		b.root = tmp.Left
		return res
	} else if b.root.Val > el {
		return deleteBST(b.root.Left, b.root, true, el)
	}
	return deleteBST(b.root.Right, b.root, false, el)
}
<commit_msg>add sync.Mutex<commit_after>package tree

import "sync"

/*
二叉查找树（Binary Search Tree）是指一棵空树或者具有下列性质的二叉树：
	1. 若任意节点的左子树不空，则左子树上所有节点的值均小于它的根节点的值；
	2. 若任意节点的右子树不空，则右子树上所有节点的值均大于它的根节点的值；
	3. 任意节点的左、右子树也分别为二叉查找树；
	4. 没有键值相等的节点。
*/
type BST struct {
	sync.Mutex
	root *TreeNode
}

// BST Construtor
func NewBST() *BST {
	return &BST{}
}

func insertBST(root *TreeNode, el int) bool {
	if root.Val == el {
		return false
	} else if root.Val > el {
		if root.Left == nil {
			root.Left = &TreeNode{Val: el}
		} else {
			return insertBST(root.Left, el)
		}
	} else {
		if root.Right == nil {
			root.Right = &TreeNode{Val: el}
		} else {
			return insertBST(root.Right, el)
		}
	}
	return true
}

// 在二叉搜索树插入节点
func (b *BST) Insert(el int) bool {
	b.Lock()
	defer b.Unlock()

	if b.root == nil {
		b.root = &TreeNode{Val: el}
		return true
	}
	return insertBST(b.root, el)
}

func searchBST(root *TreeNode, el int) bool {
	if root == nil {
		return false
	}
	if root.Val == el {
		return true
	} else if root.Val > el {
		return searchBST(root.Left, el)
	}
	return searchBST(root.Right, el)
}

// 在二叉搜索树查找一个节点
func (b *BST) Search(el int) bool {
	b.Lock()
	defer b.Unlock()

	if b.root == nil {
		return false
	}
	return searchBST(b.root, el)
}

func deleteBST(node, parent *TreeNode, isLeft bool, el int) bool {
	switch {
	case node.Val == el:
		if node.Left != nil && node.Right != nil {
			// 左右子树均不空
			q, tmp := node, node.Left
			for tmp.Right != nil {
				q, tmp = tmp, tmp.Right
			}
			node.Val = tmp.Val
			if q.Val != node.Val {
				q.Right = tmp.Left
			} else {
				q.Left = tmp.Left
			}
			return true
		}

		var tmpRes *TreeNode
		if node.Left == nil && node.Right == nil {
			// 该节点为叶子节点，直接删除
			tmpRes = nil
		} else if node.Left == nil {
			// 左子树空只需重接它的右子树
			tmpRes = node.Right
		} else if node.Right == nil {
			// 右子树空则只需重接它的左子树
			tmpRes = node.Left
		}

		if isLeft {
			parent.Left = tmpRes
		} else {
			parent.Right = tmpRes
		}
		return true
	case node.Val > el:
		if node.Left != nil {
			return deleteBST(node.Left, node, true, el)
		}
	case node.Val < el:
		if node.Right != nil {
			return deleteBST(node.Right, node, false, el)
		}
	}
	return false
}

// 在二叉查找树删除结点
func (b *BST) Delete(el int) bool {
	b.Lock()
	defer b.Unlock()

	if b.root == nil {
		return false
	}
	if b.root.Val == el {
		tmp := &TreeNode{Val: 0}
		tmp.Left = b.root
		res := deleteBST(b.root, tmp, true, el)
		b.root = tmp.Left
		return res
	} else if b.root.Val > el {
		return deleteBST(b.root.Left, b.root, true, el)
	}
	return deleteBST(b.root.Right, b.root, false, el)
}
<|endoftext|>


<commit_before>package main

import "bytes"
import "strings"
import "testing"
import "io"
import . "gopkg.in/check.v1"

type TestSuite struct{
}

func Test(t *testing.T){ TestingT(t) }

var _ = Suite(&TestSuite{})

//Expected cases

func (s *TestSuite) TestDocumentWithSnowman(c *C) {
  var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello world\n")
  c.Assert(err,IsNil)

  _, err = buf.WriteString("Do you know what a snowman is?\n")
  c.Assert(err,IsNil)

  _, err = buf.WriteString("Hello bob\n")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 1)
}

func (s *TestSuite) TestDocumentWithSnowmanTwice(c *C) {
  var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello world\n")
  c.Assert(err,IsNil)

  _, err = buf.WriteString("Do you know what a snowman snowman is?\n")
  c.Assert(err,IsNil)

  _, err = buf.WriteString("Hello bob\n")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 1)
}

func (s *TestSuite) TestDocumentWithoutSnowman(c *C) {
  var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello world\n")
  c.Assert(err,IsNil)

  _, err = buf.WriteString("Do you know what a yeti is?\n")
  c.Assert(err,IsNil)

  _, err = buf.WriteString("Hello bob\n")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 0)
}

//Edge cases
func (s *TestSuite) TestZeroLengthDocument(c *C){
 var err error
  buf := new(bytes.Buffer)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(buf.Len(), Equals, 0)
}

func (s *TestSuite) TestDocumentWithoutNewline(c *C){
 var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello snowman")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 1)
}

func (s *TestSuite) TestDocumentWithWindowsNewlines(c *C){
 var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello snowman\r\nHello bob")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  // Windows newline throws off the program, don't care about supporting this
  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 0)

}

<commit_msg>add a failure test<commit_after>package main

import "errors"
import "bytes"
import "strings"
import "testing"
import "io"
import . "gopkg.in/check.v1"

type TestSuite struct{
}

func Test(t *testing.T){ TestingT(t) }

var _ = Suite(&TestSuite{})

//Expected cases

func (s *TestSuite) TestDocumentWithSnowman(c *C) {
  var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello world\n")
  c.Assert(err,IsNil)
  _, err = buf.WriteString("Do you know what a snowman is?\n")
  c.Assert(err,IsNil)
  _, err = buf.WriteString("Hello bob\n")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 1)
}

func (s *TestSuite) TestDocumentWithSnowmanTwice(c *C) {
  var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello world\n")
  c.Assert(err,IsNil)
  _, err = buf.WriteString("Do you know what a snowman snowman is?\n")
  c.Assert(err,IsNil)
  _, err = buf.WriteString("Hello bob\n")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 1)
}

func (s *TestSuite) TestDocumentWithoutSnowman(c *C) {
  var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello world\n")
  c.Assert(err,IsNil)
  _, err = buf.WriteString("Do you know what a yeti is?\n")
  c.Assert(err,IsNil)
  _, err = buf.WriteString("Hello bob\n")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 0)
}

//Edge cases
func (s *TestSuite) TestZeroLengthDocument(c *C){
 var err error
  buf := new(bytes.Buffer)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(buf.Len(), Equals, 0)
}

func (s *TestSuite) TestDocumentWithoutNewline(c *C){
 var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello snowman")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 1)
}

func (s *TestSuite) TestDocumentWithWindowsNewlines(c *C){
 var err error
  buf := new(bytes.Buffer)

  _, err = buf.WriteString("Hello snowman\r\nHello bob")
  c.Assert(err,IsNil)

  reader := NewWinterWonderland(buf)

  output := new(bytes.Buffer)
  _, err = io.Copy(output, reader)
  c.Assert(err, IsNil)

  // Windows newline throws off the program, don't care about supporting this
  c.Assert(strings.Count(output.String(), "\u2603"), Equals, 0)

}

// Failure cases

type failureReader int //Type does not matter here

var failureReaderSentinel = errors.New("Sentinel from failure reader")

func (failureReader) Read([]byte) (int, error) {
  return 0, failureReaderSentinel
}

func (s *TestSuite) TestReaderFailure(c *C) {
  reader := NewWinterWonderland(failureReader(0))
  var output [1]byte
  _, err := reader.Read(output[:])
  c.Assert(err, Equals, failureReaderSentinel)
}

<|endoftext|>


<commit_before>package handler

import (
	"OnlineJudge/base"
	"OnlineJudge/models"
	"OnlineJudge/pbgen/api"
)

func (this *Handler) ShowProblem(response *api.ShowProblemResponse, req *api.ShowProblemRequest) {
	if err := this.OpenDB(); err != nil {
		MakeResponseError(response, this.debug, PBInternalError, err)
		return
	}
	defer this.CloseDB()

	// Get Sid
	pid, err := base.ParseSid(req.GetProblemSid())
	if err != nil {
		MakeResponseError(response, this.debug, PBBadRequest, err)
		return
	}

	// Query problem
	mp, err := models.Query_MetaProblem_By_OJName_OJPid(this.tx, pid.OJName, pid.OJPid, nil, nil)
	if err != nil {
		MakeResponseError(response, this.debug, PBInternalError, err)
		return
	}

	// QueryLanguages
	langs, err := models.Query_Languages_By_OJIdFK(this.tx, mp.OJIdFK, nil, nil)
	if err != nil {
		MakeResponseError(response, this.debug, PBInternalError, err)
		return
	}

	// Judge if authorized
	if mp.Hide != false {
		if this.session.IsLogin() == false {
			MakeResponseError(response, this.debug, PBProblemNotFound, nil)
			return
		} else {
			if this.session.GetPrivilege() != "root" {
				MakeResponseError(response, this.debug, PBProblemNotFound, nil)
				return
			}
		}
	}

	// Make response
	response.ProblemSid = req.GetProblemSid()
	problem := &api.Problem{
		Title:         mp.Title,
		TimeLimit:     int32(mp.TimeLimit),
		CaseTimeLimit: int32(mp.CaseTimeLimit),
		MemoryLimit:   int32(mp.MemoryLimit),
		Description:   mp.Description,
		Input:         mp.Input,
		Output:        mp.Output,
		SampleInput:   mp.SampleIn,
		SampleOutput:  mp.SampleOut,
		Source:        mp.Source,
		Hint:          mp.Hint,
		Hide:          mp.Hide,
	}
	response.Problem = problem
	languages := []*api.Language{}
	for _, lang := range langs {
		temp := &api.Language{
			Compiler:   lang.Compiler,
			Language:   lang.Language,
			LanguageId: lang.LangId,
		}
		languages = append(languages, temp)
	}
	response.Languages = languages

}
<commit_msg>optimize code<commit_after>package handler

import (
	"OnlineJudge/base"
	"OnlineJudge/models"
	"OnlineJudge/pbgen/api"
)

func (this *Handler) ShowProblem(response *api.ShowProblemResponse, req *api.ShowProblemRequest) {
	if err := this.OpenDB(); err != nil {
		MakeResponseError(response, this.debug, PBInternalError, err)
		return
	}
	defer this.CloseDB()

	// Get Sid
	pid, err := base.ParseSid(req.GetProblemSid())
	if err != nil {
		MakeResponseError(response, this.debug, PBBadRequest, err)
		return
	}

	// Query problem
	mp, err := models.Query_MetaProblem_By_OJName_OJPid(this.tx, pid.OJName, pid.OJPid, nil, nil)
	if err != nil {
		MakeResponseError(response, this.debug, PBInternalError, err)
		return
	}

	// QueryLanguages
	langs, err := models.Query_Languages_By_OJIdFK(this.tx, mp.OJIdFK, nil, nil)
	if err != nil {
		MakeResponseError(response, this.debug, PBInternalError, err)
		return
	}

	// Judge if authorized
	if mp.Hide != false {
		if this.session.IsLogin() == false || this.session.GetPrivilege() != "root" {
			MakeResponseError(response, this.debug, PBProblemNotFound, nil)
			return
		}
	}

	// Make response
	response.ProblemSid = req.GetProblemSid()
	problem := &api.Problem{
		Title:         mp.Title,
		TimeLimit:     int32(mp.TimeLimit),
		CaseTimeLimit: int32(mp.CaseTimeLimit),
		MemoryLimit:   int32(mp.MemoryLimit),
		Description:   mp.Description,
		Input:         mp.Input,
		Output:        mp.Output,
		SampleInput:   mp.SampleIn,
		SampleOutput:  mp.SampleOut,
		Source:        mp.Source,
		Hint:          mp.Hint,
		Hide:          mp.Hide,
	}
	response.Problem = problem
	languages := []*api.Language{}
	for _, lang := range langs {
		temp := &api.Language{
			Compiler:   lang.Compiler,
			Language:   lang.Language,
			LanguageId: lang.LangId,
		}
		languages = append(languages, temp)
	}
	response.Languages = languages

}
<|endoftext|>