<commit_before>package liblumberjack

import (
  "bytes"
  "encoding/json"
  zmq "github.com/alecthomas/gozmq"
  "log"
  "math/big"
  "syscall"
  "time"
  "compress/zlib"
  "crypto/rand"
  "sodium"
)

var context *zmq.Context

func init() {
  context, _ = zmq.NewContext()
}

// Forever Faithful Socket
type FFS struct {
  Endpoints []string // set of endpoints available to ship to

  // Socket type; zmq.REQ, etc
  SocketType zmq.SocketType

  // Various timeout values
  SendTimeout time.Duration
  RecvTimeout time.Duration

  endpoint  string      // the current endpoint in use
  socket    *zmq.Socket // the current zmq socket
  connected bool        // are we connected?
}

func (s *FFS) Send(data []byte, flags zmq.SendRecvOption) (err error) {
  for {
    s.ensure_connect()

    pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLOUT}}
    count, err := zmq.Poll(pi, s.SendTimeout)
    if count == 0 {
      // not ready in time, fail the socket and try again.
      log.Printf("%s: timed out waiting to Send(): %s\n", s.endpoint, err)
      s.fail_socket()
    } else {
      //log.Printf("%s: sending %d payload\n", s.endpoint, len(data))
      err = s.socket.Send(data, flags)
      if err != nil {
        log.Printf("%s: Failed to Send() %d byte message: %s\n",
          s.endpoint, len(data), err)
        s.fail_socket()
      } else {
        // Success!
        break
      }
    }
  }
  return
}

func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) {
  s.ensure_connect()

  pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLIN}}
  count, err := zmq.Poll(pi, s.RecvTimeout)
  if count == 0 {
    // not ready in time, fail the socket and try again.
    s.fail_socket()

    err = syscall.ETIMEDOUT
    log.Printf("%s: timed out waiting to Recv(): %s\n",
      s.endpoint, err)
    return nil, err
  } else {
    data, err = s.socket.Recv(flags)
    if err != nil {
      log.Printf("%s: Failed to Recv() %d byte message: %s\n",
        s.endpoint, len(data), err)
      s.fail_socket()
      return nil, err
    } else {
      // Success!
    }
  }
  return
}

func (s *FFS) Close() (err error) {
  err = s.socket.Close()
  if err != nil {
    return
  }

  s.socket = nil
  s.connected = false
  return nil
}

func (s *FFS) ensure_connect() {
  if s.connected {
    return
  }

  if s.SendTimeout == 0 {
    s.SendTimeout = 1 * time.Second
  }
  if s.RecvTimeout == 0 {
    s.RecvTimeout = 1 * time.Second
  }

  if s.SocketType == 0 {
    log.Panicf("No socket type set on zmq socket")
  }
  if s.socket != nil {
    s.socket.Close()
    s.socket = nil
  }

  var err error
  s.socket, err = context.NewSocket(s.SocketType)
  if err != nil {
    log.Panicf("zmq.NewSocket(%d) failed: %s\n", s.SocketType, err)
  }

  //s.socket.SetSockOptUInt64(zmq.HWM, 1)
  //s.socket.SetSockOptInt(zmq.RCVTIMEO, int(s.RecvTimeout.Nanoseconds() / 1000000))
  //s.socket.SetSockOptInt(zmq.SNDTIMEO, int(s.SendTimeout.Nanoseconds() / 1000000))

  // Abort anything in-flight on a socket that's closed.
  s.socket.SetSockOptInt(zmq.LINGER, 0)

  for !s.connected {
    var max *big.Int = big.NewInt(int64(len(s.Endpoints)))
    i, _ := rand.Int(rand.Reader, max)
    s.endpoint = s.Endpoints[i.Int64()]
    err := s.socket.Connect(s.endpoint)
    if err != nil {
      log.Printf("%s: Error connecting: %s\n", s.endpoint, err)
      time.Sleep(500 * time.Millisecond)
      continue
    }

    // No error, we're connected.
    s.connected = true
  }
}

func (s *FFS) fail_socket() {
  if !s.connected {
    return
  }
  s.Close()
}

func Publish(input chan []*FileEvent,
             registrar chan []*FileEvent,
             server_list []string,
             public_key [sodium.PUBLICKEYBYTES]byte,
             secret_key [sodium.SECRETKEYBYTES]byte,
             server_timeout time.Duration) {
  var buffer bytes.Buffer
  session := sodium.NewSession(public_key, secret_key)

  socket := FFS{
    Endpoints:   server_list,
    SocketType:  zmq.REQ,
    RecvTimeout: server_timeout,
    SendTimeout: server_timeout,
  }
  //defer socket.Close()

  for events := range input {
    // got a bunch of events, ship them out.
    log.Printf("Publisher received %d events\n", len(events))

    data, _ := json.Marshal(events)
    // TODO(sissel): check error

    // Compress it
    // A new zlib writer  is used for every payload of events so that any
    // individual payload can be decompressed alone.
    // TODO(sissel): Make compression level tunable
    compressor, _ := zlib.NewWriterLevel(&buffer, 3)
    buffer.Truncate(0)
    _, err := compressor.Write(data)
    err = compressor.Flush()
    compressor.Close()

    //log.Printf("compressed %d bytes\n", buffer.Len())
    // TODO(sissel): check err
    // TODO(sissel): implement security/encryption/etc

    // Send full payload over zeromq REQ/REP
    // TODO(sissel): check error
    //buffer.Write(data)
    ciphertext, nonce := session.Box(buffer.Bytes())

    log.Printf("plaintext: %d\n", len(data))
    log.Printf("compressed: %d\n", buffer.Len())
    log.Printf("ciphertext: %d\n", len(ciphertext))
    log.Printf("nonce: %d\n", len(nonce))

    // TODO(sissel): figure out encoding for ciphertext + nonce
    // TODO(sissel): figure out encoding for ciphertext + nonce

    // Loop forever trying to send.
    // This will cause reconnects/etc on failures automatically
    for {
      err = socket.Send(buffer.Bytes(), 0)
      if err != nil {
        continue // send failed, retry!
      }

      data, err = socket.Recv(0)
      // TODO(sissel): Figure out acknowledgement protocol? If any?
      if err == nil {
        break // success!
      }
    }

    // Tell the registrar that we've successfully sent these events
    registrar <- events
  } /* for each event payload */
} // Publish
<commit_msg>- send the nonce and ciphertext separately via zmq's SNDMORE feature<commit_after>package liblumberjack

import (
  "bytes"
  "encoding/json"
  zmq "github.com/alecthomas/gozmq"
  "log"
  "math/big"
  "syscall"
  "time"
  "compress/zlib"
  "crypto/rand"
  "sodium"
)

var context *zmq.Context

func init() {
  context, _ = zmq.NewContext()
}

// Forever Faithful Socket
type FFS struct {
  Endpoints []string // set of endpoints available to ship to

  // Socket type; zmq.REQ, etc
  SocketType zmq.SocketType

  // Various timeout values
  SendTimeout time.Duration
  RecvTimeout time.Duration

  endpoint  string      // the current endpoint in use
  socket    *zmq.Socket // the current zmq socket
  connected bool        // are we connected?
}

func (s *FFS) Send(data []byte, flags zmq.SendRecvOption) (err error) {
  for {
    s.ensure_connect()

    pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLOUT}}
    count, err := zmq.Poll(pi, s.SendTimeout)
    if count == 0 {
      // not ready in time, fail the socket and try again.
      log.Printf("%s: timed out waiting to Send(): %s\n", s.endpoint, err)
      s.fail_socket()
    } else {
      //log.Printf("%s: sending %d payload\n", s.endpoint, len(data))
      err = s.socket.Send(data, flags)
      if err != nil {
        log.Printf("%s: Failed to Send() %d byte message: %s\n",
          s.endpoint, len(data), err)
        s.fail_socket()
      } else {
        // Success!
        break
      }
    }
  }
  return
}

func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) {
  s.ensure_connect()

  pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLIN}}
  count, err := zmq.Poll(pi, s.RecvTimeout)
  if count == 0 {
    // not ready in time, fail the socket and try again.
    s.fail_socket()

    err = syscall.ETIMEDOUT
    log.Printf("%s: timed out waiting to Recv(): %s\n",
      s.endpoint, err)
    return nil, err
  } else {
    data, err = s.socket.Recv(flags)
    if err != nil {
      log.Printf("%s: Failed to Recv() %d byte message: %s\n",
        s.endpoint, len(data), err)
      s.fail_socket()
      return nil, err
    } else {
      // Success!
    }
  }
  return
}

func (s *FFS) Close() (err error) {
  err = s.socket.Close()
  if err != nil {
    return
  }

  s.socket = nil
  s.connected = false
  return nil
}

func (s *FFS) ensure_connect() {
  if s.connected {
    return
  }

  if s.SendTimeout == 0 {
    s.SendTimeout = 1 * time.Second
  }
  if s.RecvTimeout == 0 {
    s.RecvTimeout = 1 * time.Second
  }

  if s.SocketType == 0 {
    log.Panicf("No socket type set on zmq socket")
  }
  if s.socket != nil {
    s.socket.Close()
    s.socket = nil
  }

  var err error
  s.socket, err = context.NewSocket(s.SocketType)
  if err != nil {
    log.Panicf("zmq.NewSocket(%d) failed: %s\n", s.SocketType, err)
  }

  //s.socket.SetSockOptUInt64(zmq.HWM, 1)
  //s.socket.SetSockOptInt(zmq.RCVTIMEO, int(s.RecvTimeout.Nanoseconds() / 1000000))
  //s.socket.SetSockOptInt(zmq.SNDTIMEO, int(s.SendTimeout.Nanoseconds() / 1000000))

  // Abort anything in-flight on a socket that's closed.
  s.socket.SetSockOptInt(zmq.LINGER, 0)

  for !s.connected {
    var max *big.Int = big.NewInt(int64(len(s.Endpoints)))
    i, _ := rand.Int(rand.Reader, max)
    s.endpoint = s.Endpoints[i.Int64()]
    err := s.socket.Connect(s.endpoint)
    if err != nil {
      log.Printf("%s: Error connecting: %s\n", s.endpoint, err)
      time.Sleep(500 * time.Millisecond)
      continue
    }

    // No error, we're connected.
    s.connected = true
  }
}

func (s *FFS) fail_socket() {
  if !s.connected {
    return
  }
  s.Close()
}

func Publish(input chan []*FileEvent,
             registrar chan []*FileEvent,
             server_list []string,
             public_key [sodium.PUBLICKEYBYTES]byte,
             secret_key [sodium.SECRETKEYBYTES]byte,
             server_timeout time.Duration) {
  var buffer bytes.Buffer
  session := sodium.NewSession(public_key, secret_key)

  socket := FFS{
    Endpoints:   server_list,
    SocketType:  zmq.REQ,
    RecvTimeout: server_timeout,
    SendTimeout: server_timeout,
  }
  //defer socket.Close()

  for events := range input {
    // got a bunch of events, ship them out.
    log.Printf("Publisher received %d events\n", len(events))

    data, _ := json.Marshal(events)
    // TODO(sissel): check error

    // Compress it
    // A new zlib writer  is used for every payload of events so that any
    // individual payload can be decompressed alone.
    // TODO(sissel): Make compression level tunable
    compressor, _ := zlib.NewWriterLevel(&buffer, 3)
    buffer.Truncate(0)
    _, err := compressor.Write(data)
    err = compressor.Flush()
    compressor.Close()

    //log.Printf("compressed %d bytes\n", buffer.Len())
    // TODO(sissel): check err
    // TODO(sissel): implement security/encryption/etc

    // Send full payload over zeromq REQ/REP
    // TODO(sissel): check error
    //buffer.Write(data)
    ciphertext, nonce := session.Box(buffer.Bytes())

    //log.Printf("plaintext: %d\n", len(data))
    //log.Printf("compressed: %d\n", buffer.Len())
    //log.Printf("ciphertext: %d %v\n", len(ciphertext), ciphertext[:20])
    //log.Printf("nonce: %d\n", len(nonce))

    // TODO(sissel): figure out encoding for ciphertext + nonce
    // TODO(sissel): figure out encoding for ciphertext + nonce

    // Loop forever trying to send.
    // This will cause reconnects/etc on failures automatically
    for {
      err = socket.Send(nonce[:], zmq.SNDMORE)
      if err != nil {
        continue // send failed, retry!
      }
      err = socket.Send(ciphertext, 0)
      if err != nil {
        continue // send failed, retry!
      }

      data, err = socket.Recv(0)
      // TODO(sissel): Figure out acknowledgement protocol? If any?
      if err == nil {
        break // success!
      }
    }

    // Tell the registrar that we've successfully sent these events
    //registrar <- events
  } /* for each event payload */
} // Publish
<|endoftext|>


<commit_before>// Copyright 2020 Martin Holst Swende. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the COPYING file.
//

// Package math provides integer math utilities.

package uint256

import (
	"math/big"
	"math/bits"
)

const (
	maxWords = 256 / bits.UintSize // number of big.Words in 256-bit
)

// NewFromBig creates new Int from big.Int.
func NewFromBig(b *big.Int) (*Int, bool) {
	z := &Int{}
	overflow := z.SetFromBig(b)
	return z, overflow
}

// SetFromBig
// TODO: finish implementation by adding 32-bit platform support,
// ensure we have sufficient testing, esp for negative bigints
func (z *Int) SetFromBig(b *big.Int) bool {
	z.Clear()
	words := b.Bits()
	overflow := len(words) > maxWords

	switch maxWords { // Compile-time check.
	case 4: // 64-bit architectures.
		if len(words) > 0 {
			z[0] = uint64(words[0])
			if len(words) > 1 {
				z[1] = uint64(words[1])
				if len(words) > 2 {
					z[2] = uint64(words[2])
					if len(words) > 3 {
						z[3] = uint64(words[3])
					}
				}
			}
		}
	default:
		panic("unsupported architecture")
	}

	if b.Sign() == -1 {
		z.Neg()
	}
	return overflow
}
<commit_msg>conversion: Implement NewFromBig() for 32-bit architectures<commit_after>// Copyright 2020 Martin Holst Swende. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the COPYING file.
//

// Package math provides integer math utilities.

package uint256

import (
	"math/big"
	"math/bits"
)

const (
	maxWords = 256 / bits.UintSize // number of big.Words in 256-bit
)

// NewFromBig creates new Int from big.Int.
func NewFromBig(b *big.Int) (*Int, bool) {
	z := &Int{}
	overflow := z.SetFromBig(b)
	return z, overflow
}

// SetFromBig converts a big.Int to Int and sets the value to z.
// TODO: Ensure we have sufficient testing, esp for negative bigints.
func (z *Int) SetFromBig(b *big.Int) bool {
	z.Clear()
	words := b.Bits()
	overflow := len(words) > maxWords

	switch maxWords { // Compile-time check.
	case 4: // 64-bit architectures.
		if len(words) > 0 {
			z[0] = uint64(words[0])
			if len(words) > 1 {
				z[1] = uint64(words[1])
				if len(words) > 2 {
					z[2] = uint64(words[2])
					if len(words) > 3 {
						z[3] = uint64(words[3])
					}
				}
			}
		}
	case 8: // 32-bit architectures.
		numWords := len(words)
		if overflow {
			numWords = maxWords
		}
		for i := 0; i < numWords; i++ {
			if i%2 == 0 {
				z[i/2] = uint64(words[i])
			} else {
				z[i/2] |= uint64(words[i]) << 32
			}
		}
	default:
		panic("unsupported architecture")
	}

	if b.Sign() == -1 {
		z.Neg()
	}
	return overflow
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/julienschmidt/httprouter"
)

func TestOpenBuildQueue(t *testing.T) {
	req, _ := http.NewRequest("POST", "http://example.com", nil)

	w := httptest.NewRecorder()

	getShutdownChan = make(chan Shutdown, 1)
	setShutdownChan = make(chan Shutdown, 1)

	// any incumbent value will do to avoid blocking on the channel read
	getShutdownChan <- "anyvalue"

	ShutdownHandler(w, req, []httprouter.Param{
		httprouter.Param{Key: "state", Value: string(BUILD_QUEUE_OPEN)},
	},
	)

	if w.Code != 200 {
		t.Fatalf("Want 200 but got %d\n", w.Code)
	}

	state := <-setShutdownChan
	if state != BUILD_QUEUE_OPEN {
		t.Fatalf("Want open but got %s\n", state)
	}
}

func TestCloseBuildQueue(t *testing.T) {
	req, _ := http.NewRequest("POST", "http://example.com", nil)

	w := httptest.NewRecorder()

	getShutdownChan = make(chan Shutdown, 1)
	setShutdownChan = make(chan Shutdown, 1)

	// any incumbent value will do to avoid blocking on the channel read
	getShutdownChan <- "anyvalue"

	ShutdownHandler(w, req, []httprouter.Param{
		httprouter.Param{Key: "state", Value: string(BUILD_QUEUE_CLOSE)},
	},
	)

	if w.Code != 200 {
		t.Fatalf("Want 200 but got %d\n", w.Code)
	}

	state := <-setShutdownChan
	if state != BUILD_QUEUE_CLOSE {
		t.Fatalf("Want open but got %s\n", state)
	}
}

func TestInvalidShutdownBuildQueue(t *testing.T) {
	req, _ := http.NewRequest("POST", "http://example.com", nil)

	w := httptest.NewRecorder()

	getShutdownChan = make(chan Shutdown, 1)
	setShutdownChan = make(chan Shutdown, 1)

	// any incumbent value will do to avoid blocking on the channel read
	getShutdownChan <- "anyvalue"

	ShutdownHandler(w, req, []httprouter.Param{
		httprouter.Param{Key: "state", Value: string("nope")},
	},
	)

	if w.Code != 400 {
		t.Fatalf("Want 400 but got %d\n", w.Code)
	}
}
<commit_msg>improve unit test of shutdown build queue<commit_after>package main

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/julienschmidt/httprouter"
)

func TestShutdownBuildQueueOpen(t *testing.T) {
	req, _ := http.NewRequest("POST", "http://example.com", nil)

	getShutdownChan = make(chan Shutdown, 1)
	setShutdownChan = make(chan Shutdown, 1)
	getShutdownChan <- "any incumbent value will do to avoid blocking on the channel read"

	w := httptest.NewRecorder()
	ShutdownHandler(w, req, []httprouter.Param{
		httprouter.Param{Key: "state", Value: string(BUILD_QUEUE_OPEN)},
	},
	)

	if w.Code != 200 {
		t.Fatalf("Want 200 but got %d\n", w.Code)
	}

	state := <-setShutdownChan
	if state != BUILD_QUEUE_OPEN {
		t.Fatalf("Want open but got %s\n", state)
	}
}

func TestShutdownBuildQueueClose(t *testing.T) {
	req, _ := http.NewRequest("POST", "http://example.com", nil)

	getShutdownChan = make(chan Shutdown, 1)
	setShutdownChan = make(chan Shutdown, 1)
	getShutdownChan <- "any incumbent value will do to avoid blocking on the channel read"

	w := httptest.NewRecorder()
	ShutdownHandler(w, req, []httprouter.Param{
		httprouter.Param{Key: "state", Value: string(BUILD_QUEUE_CLOSE)},
	},
	)

	if w.Code != 200 {
		t.Fatalf("Want 200 but got %d\n", w.Code)
	}

	state := <-setShutdownChan
	if state != BUILD_QUEUE_CLOSE {
		t.Fatalf("Want open but got %s\n", state)
	}
}

func TestShutdownBuildQueueInvalid(t *testing.T) {
	req, _ := http.NewRequest("POST", "http://example.com", nil)

	getShutdownChan = make(chan Shutdown, 1)
	setShutdownChan = make(chan Shutdown, 1)
	getShutdownChan <- "any incumbent value will do to avoid blocking on the channel read"

	w := httptest.NewRecorder()
	ShutdownHandler(w, req, []httprouter.Param{
		httprouter.Param{Key: "state", Value: string("nope")},
	},
	)

	if w.Code != 400 {
		t.Fatalf("Want 400 but got %d\n", w.Code)
	}
}
<|endoftext|>


<commit_before>package copy

import (
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
)

// File Meta data representation
type Meta struct {
	Id                      string                  `json:"id,omitempty"`
	Path                    string                  `json:"path,omitempty"`
	Name                    string                  `json:"name,omitempty"`
	LinkName                string                  `json:"link_name,omitempty"`
	Token                   string                  `json:"token,omitempty"`
	Permissions             string                  `json:"permissions,omitempty"`
	Public                  bool                    `json:"public,omitempty"`
	Type                    string                  `json:"type,omitempty"`
	Size                    int                     `json:"size,omitempty"`
	DateLastSynced          int                     `json:"date_last_synced,omitempty"`
	ModifiedTime            int                     `json:"modified_time,omitempty"`
	Stub                    bool                    `json:"stub,omitempty"`
	Share                   bool                    `json:"share,omitempty"`
	Children                []Meta                  `json:"children,omitempty"` // Inception :D
	Counts                  Count                   `json:"counts,omitempty"`   // Array? (sometimes? ask copy.com)
	RecipientConfirmed      bool                    `json:"recipient_confirmed",omitempty"`
	MimeType                string                  `json:"mime_type",omitempty"`
	Syncing                 bool                    `json:"syncing",omitempty"`
	ObjectAvailable         bool                    `json:"object_available,omitempty"`
	Links                   []Link                  `json:"links,omitempty"`
	Revisions               []Revision              `json:"revisions,omitempty"`
	Url                     string                  `json:"url,omitempty"`
	RevisionId              int                     `json:"revision_id,omitempty"`
	Thumb                   string                  `json:"thumb,omitempty"`
	ThumbOriginalDimensions ThumbOriginalDimensions `json:"thumb_original_dimensions,omitempty"`
	ChildrenCount           int                     `json:"children_count",omitempty"`
	Revision                int                     `json:"revision",omitempty"`
	ListIndex               int                     `json:"list_index",omitempty"`
}

type Count struct {
	New    int `json:"new,omitempty"`
	Viewed int `json:"viewed,omitempty"`
	Hidden int `json:"hidden,omitempty"`
}

type Link struct {
	Id                   string      `json:"id,omitempty"`
	Public               bool        `json:"public,omitempty"`
	Expires              bool        `json:"expires,omitempty"`
	Expired              bool        `json:"expired,omitempty"`
	Url                  string      `json:"url,omitempty"`
	UrlShort             string      `json:"url_short,omitempty"`
	Recipients           []Recipient `json:"recipients,omitempty"`
	CreatorId            string      `json:"creator_id,omitempty"`
	ConfirmationRequired bool        `json:"confirmation_required,omitempty"`
}
type Recipient struct {
	ContactType   string  `json:"contact_type,omitempty"`
	ContactId     string  `json:"contact_id,omitempty"`
	ContactSource string  `json:"contact_source,omitempty"`
	UserId        string  `json:"user_id,omitempty"`
	FirstName     string  `json:"first_name,omitempty"`
	LastName      string  `json:"last_name,omitempty"`
	Email         string  `json:"email,omitempty"`
	Permissions   string  `json:"permissions,omitempty"`
	Emails        []Email `json:"emails,omitempty"` // In users.go
}

type ThumbOriginalDimensions struct {
	Width  int `json:"width,omitempty"`
	Height int `json:"Height,omitempty"`
}

type Revision struct {
	RevisionId   string  `json:"revision_id,omitempty"`
	ModifiedTime string  `json:"modified_time,omitempty"`
	Size         int     `json:"size,omitempty"`
	Latest       bool    `json:"latest,omitempty"`
	Conflict     int     `json:"conflict,omitempty"`
	Id           string  `json:"id,omitempty"`
	Type         string  `json:"type,omitempty"`
	Creator      Creator `json:"creator,omitempty"`
}

type Creator struct {
	UserId      string `json:"user_id,omitempty"`
	CreatedTime int    `json:"created_time,omitempty"`
	Email       string `json:"email,omitempty"`
	FirstName   string `json:"first_name,omitempty"`
	LastName    string `json:"last_name,omitempty"`
	Confirmed   bool   `json:"confirmed,omitempty"`
}

type FileService struct {
	client *Client
}

var (
	//Options
	overwriteOption = "overwrite=%t"
	nameOption      = "name=%v"
	pathOption      = "path=%v"
	sizeOption      = "size=%d"

	// Meta paths
	metaTopLevelSuffix  = "meta"                                                        // http.../meta/PATH
	firstLevelSuffix    = strings.Join([]string{metaTopLevelSuffix, "copy"}, "/")       // http.../meta/copy
	getMetaSuffix       = strings.Join([]string{firstLevelSuffix, "%v"}, "/")           // http.../meta/copy/PATH
	listRevisionsSuffix = strings.Join([]string{firstLevelSuffix, "%v/@activity"}, "/") // http.../meta/copy/PATH/@activity
	revisionSuffix      = strings.Join([]string{listRevisionsSuffix, "@time:%d"}, "/")  // http.../meta/copy/PATH/@activity/@time:TIME

	// File paths
	filesTopLevelSuffix  = "files"
	filesCreateSuffix    = strings.Join([]string{filesTopLevelSuffix, "/%v?", overwriteOption}, "")                  // http.../files/PATH?overwrite=FLAG
	filesRenameSuffix    = strings.Join([]string{filesTopLevelSuffix, "/%v?", nameOption, "&", overwriteOption}, "") // http.../files/PATH?name=NEWFILENAME&overwrite=FLAG
	filesMoveSuffix      = strings.Join([]string{filesTopLevelSuffix, "/%v?", pathOption, "&", overwriteOption}, "") // http.../files/PATH?overwrite=FLAG
	filesThumbnailSuffix = strings.Join([]string{filesTopLevelSuffix, "/%v?", sizeOption}, "")                       // http.../files/PATH?size=SIZE
)

func NewFileService(client *Client) *FileService {
	fs := new(FileService)
	fs.client = client
	return fs
}

// Returns the top level metadata (this is root folder, cannot change, see docs)
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetTopLevelMeta() (*Meta, error) {
	meta := new(Meta)
	resp, err := fs.client.DoRequestDecoding("GET", metaTopLevelSuffix, nil, meta)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return meta, nil
}

// Returns the metadata of a file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetMeta(path string) (*Meta, error) {

	path = strings.Trim(path, "/")

	meta := new(Meta)
	resp, err := fs.client.DoRequestDecoding("GET", fmt.Sprintf(getMetaSuffix, path), nil, meta)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return meta, nil
}

// Returns all the metadata revisions of a file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) ListRevisionsMeta(path string) ([]Revision, error) {
	meta := new(Meta)
	resp, err := fs.client.DoRequestDecoding("GET", fmt.Sprintf(listRevisionsSuffix, path), nil, meta)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return meta.Revisions, nil
}

// Returns the metadata in an specified revision
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetRevisionMeta(path string, time int) (*Meta, error) {
	return nil, nil
}

// Returns the file content. the user NEEDS TO CLOSE the buffer after using it
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetFile(path string) (io.ReadCloser, error) {

	resp, err := fs.client.DoRequestContent(strings.Join([]string{filesTopLevelSuffix, path}, "/"))

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return resp.Body, nil
}

// Deletes the file content
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) DeleteFile(path string) error {
	return nil
}

// Uploads the file. Loads the file from the file path and uploads to the
// uploadPath.
// For example:
//   filePath: /home/slok/myFile.txt
//   UploadPath: test/uploads/something.txt
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) UploadFile(filePath, uploadPath string, overwrite bool) error {

	// Sanitize path
	uploadPath = strings.Trim(uploadPath, "/")

	// Get upload filename
	filename := filepath.Base(uploadPath)

	if filename == "" {
		return errors.New("Wrong uploadPath")
	}

	// Get upload path
	uploadPath = filepath.Dir(uploadPath)

	if uploadPath == "." { // Check if is at root, if so delete the point returned by Dir
		uploadPath = ""
	}

	// Sanitize path again
	uploadPath = strings.Trim(uploadPath, "/")

	// Create final path
	uploadPath = fmt.Sprintf(filesCreateSuffix, uploadPath, overwrite)

	res, err := fs.client.DoRequestMultipart(filePath, uploadPath, filename)

	if err != nil {
		return err
	}

	if res.StatusCode >= 400 { // 400s and 500s
		return errors.New(fmt.Sprintf("Client response: %d", res.StatusCode))
	}

	return nil
}

// Renames the file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) RenameFile(path string, newName string, overwrite bool) error {

	path = strings.Trim(path, "/")

	resp, err := fs.client.DoRequestDecoding("PUT", fmt.Sprintf(filesRenameSuffix, path, newName, overwrite), nil, nil)

	if err != nil {
		return err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return nil
}

// Moves the file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) MoveFile(path string, newPath string) error {
	return nil
}

// Creates a directory
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) CreateDirectory(path string) error {
	return nil
}
<commit_msg>Refactor rename file<commit_after>package copy

import (
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
)

// File Meta data representation
type Meta struct {
	Id                      string                  `json:"id,omitempty"`
	Path                    string                  `json:"path,omitempty"`
	Name                    string                  `json:"name,omitempty"`
	LinkName                string                  `json:"link_name,omitempty"`
	Token                   string                  `json:"token,omitempty"`
	Permissions             string                  `json:"permissions,omitempty"`
	Public                  bool                    `json:"public,omitempty"`
	Type                    string                  `json:"type,omitempty"`
	Size                    int                     `json:"size,omitempty"`
	DateLastSynced          int                     `json:"date_last_synced,omitempty"`
	ModifiedTime            int                     `json:"modified_time,omitempty"`
	Stub                    bool                    `json:"stub,omitempty"`
	Share                   bool                    `json:"share,omitempty"`
	Children                []Meta                  `json:"children,omitempty"` // Inception :D
	Counts                  Count                   `json:"counts,omitempty"`   // Array? (sometimes? ask copy.com)
	RecipientConfirmed      bool                    `json:"recipient_confirmed",omitempty"`
	MimeType                string                  `json:"mime_type",omitempty"`
	Syncing                 bool                    `json:"syncing",omitempty"`
	ObjectAvailable         bool                    `json:"object_available,omitempty"`
	Links                   []Link                  `json:"links,omitempty"`
	Revisions               []Revision              `json:"revisions,omitempty"`
	Url                     string                  `json:"url,omitempty"`
	RevisionId              int                     `json:"revision_id,omitempty"`
	Thumb                   string                  `json:"thumb,omitempty"`
	ThumbOriginalDimensions ThumbOriginalDimensions `json:"thumb_original_dimensions,omitempty"`
	ChildrenCount           int                     `json:"children_count",omitempty"`
	Revision                int                     `json:"revision",omitempty"`
	ListIndex               int                     `json:"list_index",omitempty"`
}

type Count struct {
	New    int `json:"new,omitempty"`
	Viewed int `json:"viewed,omitempty"`
	Hidden int `json:"hidden,omitempty"`
}

type Link struct {
	Id                   string      `json:"id,omitempty"`
	Public               bool        `json:"public,omitempty"`
	Expires              bool        `json:"expires,omitempty"`
	Expired              bool        `json:"expired,omitempty"`
	Url                  string      `json:"url,omitempty"`
	UrlShort             string      `json:"url_short,omitempty"`
	Recipients           []Recipient `json:"recipients,omitempty"`
	CreatorId            string      `json:"creator_id,omitempty"`
	ConfirmationRequired bool        `json:"confirmation_required,omitempty"`
}
type Recipient struct {
	ContactType   string  `json:"contact_type,omitempty"`
	ContactId     string  `json:"contact_id,omitempty"`
	ContactSource string  `json:"contact_source,omitempty"`
	UserId        string  `json:"user_id,omitempty"`
	FirstName     string  `json:"first_name,omitempty"`
	LastName      string  `json:"last_name,omitempty"`
	Email         string  `json:"email,omitempty"`
	Permissions   string  `json:"permissions,omitempty"`
	Emails        []Email `json:"emails,omitempty"` // In users.go
}

type ThumbOriginalDimensions struct {
	Width  int `json:"width,omitempty"`
	Height int `json:"Height,omitempty"`
}

type Revision struct {
	RevisionId   string  `json:"revision_id,omitempty"`
	ModifiedTime string  `json:"modified_time,omitempty"`
	Size         int     `json:"size,omitempty"`
	Latest       bool    `json:"latest,omitempty"`
	Conflict     int     `json:"conflict,omitempty"`
	Id           string  `json:"id,omitempty"`
	Type         string  `json:"type,omitempty"`
	Creator      Creator `json:"creator,omitempty"`
}

type Creator struct {
	UserId      string `json:"user_id,omitempty"`
	CreatedTime int    `json:"created_time,omitempty"`
	Email       string `json:"email,omitempty"`
	FirstName   string `json:"first_name,omitempty"`
	LastName    string `json:"last_name,omitempty"`
	Confirmed   bool   `json:"confirmed,omitempty"`
}

type FileService struct {
	client *Client
}

var (
	//Options
	overwriteOption = "overwrite=%t"
	nameOption      = "name=%v"
	pathOption      = "path=%v"
	sizeOption      = "size=%d"

	// Meta paths
	metaTopLevelSuffix  = "meta"                                                        // http.../meta/PATH
	firstLevelSuffix    = strings.Join([]string{metaTopLevelSuffix, "copy"}, "/")       // http.../meta/copy
	getMetaSuffix       = strings.Join([]string{firstLevelSuffix, "%v"}, "/")           // http.../meta/copy/PATH
	listRevisionsSuffix = strings.Join([]string{firstLevelSuffix, "%v/@activity"}, "/") // http.../meta/copy/PATH/@activity
	revisionSuffix      = strings.Join([]string{listRevisionsSuffix, "@time:%d"}, "/")  // http.../meta/copy/PATH/@activity/@time:TIME

	// File paths
	filesTopLevelSuffix  = "files"
	filesCreateSuffix    = strings.Join([]string{filesTopLevelSuffix, "/%v?", overwriteOption}, "")                  // http.../files/PATH?overwrite=FLAG
	filesRenameSuffix    = strings.Join([]string{filesTopLevelSuffix, "/%v?", nameOption, "&", overwriteOption}, "") // http.../files/PATH?name=NEWFILENAME&overwrite=FLAG
	filesMoveSuffix      = strings.Join([]string{filesTopLevelSuffix, "/%v?", pathOption, "&", overwriteOption}, "") // http.../files/PATH?overwrite=FLAG
	filesThumbnailSuffix = strings.Join([]string{filesTopLevelSuffix, "/%v?", sizeOption}, "")                       // http.../files/PATH?size=SIZE
)

func NewFileService(client *Client) *FileService {
	fs := new(FileService)
	fs.client = client
	return fs
}

// Returns the top level metadata (this is root folder, cannot change, see docs)
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetTopLevelMeta() (*Meta, error) {
	meta := new(Meta)
	resp, err := fs.client.DoRequestDecoding("GET", metaTopLevelSuffix, nil, meta)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return meta, nil
}

// Returns the metadata of a file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetMeta(path string) (*Meta, error) {

	path = strings.Trim(path, "/")

	meta := new(Meta)
	resp, err := fs.client.DoRequestDecoding("GET", fmt.Sprintf(getMetaSuffix, path), nil, meta)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return meta, nil
}

// Returns all the metadata revisions of a file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) ListRevisionsMeta(path string) ([]Revision, error) {
	meta := new(Meta)
	resp, err := fs.client.DoRequestDecoding("GET", fmt.Sprintf(listRevisionsSuffix, path), nil, meta)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return meta.Revisions, nil
}

// Returns the metadata in an specified revision
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetRevisionMeta(path string, time int) (*Meta, error) {
	return nil, nil
}

// Returns the file content. the user NEEDS TO CLOSE the buffer after using it
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetFile(path string) (io.ReadCloser, error) {

	resp, err := fs.client.DoRequestContent(strings.Join([]string{filesTopLevelSuffix, path}, "/"))

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return nil, errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return resp.Body, nil
}

// Deletes the file content
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) DeleteFile(path string) error {
	return nil
}

// Uploads the file. Loads the file from the file path and uploads to the
// uploadPath.
// For example:
//   filePath: /home/slok/myFile.txt
//   UploadPath: test/uploads/something.txt
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) UploadFile(filePath, uploadPath string, overwrite bool) error {

	// Sanitize path
	uploadPath = strings.Trim(uploadPath, "/")

	// Get upload filename
	filename := filepath.Base(uploadPath)

	if filename == "" {
		return errors.New("Wrong uploadPath")
	}

	// Get upload path
	uploadPath = filepath.Dir(uploadPath)

	if uploadPath == "." { // Check if is at root, if so delete the point returned by Dir
		uploadPath = ""
	}

	// Sanitize path again
	uploadPath = strings.Trim(uploadPath, "/")

	// Create final path
	uploadPath = fmt.Sprintf(filesCreateSuffix, uploadPath, overwrite)

	res, err := fs.client.DoRequestMultipart(filePath, uploadPath, filename)

	if err != nil {
		return err
	}

	if res.StatusCode >= 400 { // 400s and 500s
		return errors.New(fmt.Sprintf("Client response: %d", res.StatusCode))
	}

	return nil
}

// Renames the file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) RenameFile(path string, newName string, overwrite bool) error {
	path = strings.Trim(path, "/")
	return fs.moveOrRenameFile(fmt.Sprintf(filesRenameSuffix, path, newName, overwrite))
}

// Moves the file
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) MoveFile(path string, newPath string, overwrite bool) error {
	return nil
}

// Move and rename calls are similar, wrap in this function for convienence
func (fs *FileService) moveOrRenameFile(finalUrl string) error {

	resp, err := fs.client.DoRequestDecoding("PUT", finalUrl, nil, nil)

	if err != nil {
		return err
	}

	if resp.StatusCode >= 400 { // 400s and 500s
		return errors.New(fmt.Sprintf("Client response: %d", resp.StatusCode))
	}

	return nil
}

// Creates a directory
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) CreateDirectory(path string) error {
	return nil
}
<|endoftext|>


<commit_before>package core

type Facts map[string]string

func (f Facts) OS() string {
	return f["OS"]
}

func (f Facts) Arch() string {
	arch, ok := f["MACH"]
	if !ok {
		return ""
	}

	if arch == "x86_64" {
		return "amd64"
	}

	return arch
}

func (f Facts) Home() string {
	return f["HOME"]
}
<commit_msg>added 32bit architecture<commit_after>package core

type Facts map[string]string

func (f Facts) OS() string {
	return f["OS"]
}

func (f Facts) Arch() string {
	arch, ok := f["MACH"]
	if !ok {
		return ""
	}

	if arch == "x86_64" {
		return "amd64"
	}

	if arch == "i686" {
		return "386"
	}

	return arch
}

func (f Facts) Home() string {
	return f["HOME"]
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"

	"discovery-artifact-manager/common/environment"
)

// discoURL specifies a URL for the live Discovery service index.
const discoURL = "https://www.googleapis.com/discovery/v1/apis"

type apiInfo struct {
	Name, Version, DiscoveryRestUrl string
}

// UpdateDiscos updates local Discovery doc files for all APIs indexed by the live Discovery
// service, in a top-level directory 'discoveries', which must exist.
func UpdateDiscos() error {
	root, err := environment.RepoRoot()
	if err != nil {
		return fmt.Errorf("Error finding repository root directory: %v", err)
	}
	discoPath := path.Join(root, "discoveries")
	info, err := os.Stat(discoPath)
	if os.IsNotExist(err) {
		return fmt.Errorf("Error finding path for Discovery docs: %v", discoPath)
	}
	if err := os.RemoveAll(discoPath); err != nil {
		return fmt.Errorf("Error removing old Discovery docs: %v", err)
	}
	perm := info.Mode()
	if err := os.MkdirAll(discoPath, perm); err != nil {
		return fmt.Errorf("Error re-initializing path for Discovery docs: %v", err)
	}

	fmt.Println(fmt.Sprintf("Fetching Discovery doc index from %v ...", discoURL))
	response, err := http.Get(discoURL)
	if err != nil {
		return fmt.Errorf("Error fetching Discovery doc index: %v", err)
	}
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return fmt.Errorf("Error reading Discovery doc index: %v", err)
	}

	fmt.Println("Reading Discovery doc index ...")
	var index struct {
		Items []apiInfo
	}
	if err := json.Unmarshal(body, &index); err != nil {
		return fmt.Errorf("Error parsing Discovery doc index: %v", err)
	}

	fmt.Println(fmt.Sprintf("Updating local Discovery docs in %v:", discoPath))
	var failed []apiInfo
	// Make Discovery doc file permissions like parent directory (no execute)
	perm &= 0666
	for _, api := range index.Items {
		if err := UpdateAPI(api, discoPath, perm); err != nil {
			failed = append(failed, api)
			fmt.Println(err)
		}
	}
	if len(failed) > 0 {
		return errors.New(fmt.Sprintf("Error updating some APIs: %v", failed))
	}
	return nil
}

// UpdateAPI updates the local Discovery doc file for an API indexed by the live Discovery service.
func UpdateAPI(api apiInfo, discoPath string, perm os.FileMode) error {
	fmt.Println(fmt.Sprintf("Updating API: %v %v ...", api.Name, api.Version))
	response, err := http.Get(api.DiscoveryRestUrl)
	if err != nil {
		return fmt.Errorf("Error downloading Discovery doc from %v: %v", api.DiscoveryRestUrl, err)
	}
	defer response.Body.Close()

	filename := path.Join(discoPath, api.Name+"."+api.Version+".json")
	disco, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
	if err != nil {
		return fmt.Errorf("Error creating local discovery doc file: %v", filename)
	}
	defer disco.Close()
	if _, err := io.Copy(disco, response.Body); err != nil {
		return fmt.Errorf("Error writing local Discovery doc file: %v", filename)
	}
	return nil
}
<commit_msg>Parallelize Discovery doc update<commit_after>package common

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"

	"discovery-artifact-manager/common/environment"
)

// discoURL specifies a URL for the live Discovery service index.
const discoURL = "https://www.googleapis.com/discovery/v1/apis"

type apiInfo struct {
	Name, Version, DiscoveryRestUrl string
}

// UpdateDiscos updates local Discovery doc files for all APIs indexed by the live Discovery
// service, in a top-level directory 'discoveries', which must exist.
func UpdateDiscos() error {
	root, err := environment.RepoRoot()
	if err != nil {
		return fmt.Errorf("Error finding repository root directory: %v", err)
	}
	discoPath := path.Join(root, "discoveries")
	info, err := os.Stat(discoPath)
	if os.IsNotExist(err) {
		return fmt.Errorf("Error finding path for Discovery docs: %v", discoPath)
	}
	if err := os.RemoveAll(discoPath); err != nil {
		return fmt.Errorf("Error removing old Discovery docs: %v", err)
	}
	perm := info.Mode()
	if err := os.MkdirAll(discoPath, perm); err != nil {
		return fmt.Errorf("Error re-initializing path for Discovery docs: %v", err)
	}

	fmt.Println(fmt.Sprintf("Fetching Discovery doc index from %v ...", discoURL))
	response, err := http.Get(discoURL)
	if err != nil {
		return fmt.Errorf("Error fetching Discovery doc index: %v", err)
	}
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return fmt.Errorf("Error reading Discovery doc index: %v", err)
	}

	fmt.Println("Reading Discovery doc index ...")
	var index struct {
		Items []apiInfo
	}
	if err := json.Unmarshal(body, &index); err != nil {
		return fmt.Errorf("Error parsing Discovery doc index: %v", err)
	}

	fmt.Println(fmt.Sprintf("Updating local Discovery docs in %v:", discoPath))
	// Make Discovery doc file permissions like parent directory (no execute)
	perm &= 0666
	var update sync.WaitGroup
	failedc := make(chan apiInfo, len(index.Items))
	for _, api := range index.Items {
		update.Add(1)
		go func(api apiInfo) {
			defer update.Done()
			if err := UpdateAPI(api, discoPath, perm); err != nil {
				failedc <- api
				fmt.Println(err)
			}
		}(api)
	}
	update.Wait()
	close(failedc)
	var failed []apiInfo
	for api := range failedc {
		failed = append(failed, api)
	}
	if len(failed) > 0 {
		return errors.New(fmt.Sprintf("Error updating some APIs: %v", failed))
	}
	return nil
}

// UpdateAPI updates the local Discovery doc file for an API indexed by the live Discovery service.
func UpdateAPI(api apiInfo, discoPath string, perm os.FileMode) error {
	fmt.Println(fmt.Sprintf("Updating API: %v %v ...", api.Name, api.Version))
	response, err := http.Get(api.DiscoveryRestUrl)
	if err != nil {
		return fmt.Errorf("Error downloading Discovery doc from %v: %v", api.DiscoveryRestUrl, err)
	}
	defer response.Body.Close()

	filename := path.Join(discoPath, api.Name+"."+api.Version+".json")
	disco, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
	if err != nil {
		return fmt.Errorf("Error creating local discovery doc file: %v", filename)
	}
	defer disco.Close()
	if _, err := io.Copy(disco, response.Body); err != nil {
		return fmt.Errorf("Error writing local Discovery doc file: %v", filename)
	}
	return nil
}
<|endoftext|>


<commit_before>package webserver

import "testing"
import "github.com/cskr/pubsub"

func TestRace(t *testing.T) {

	evPS := pubsub.New(10)

	s := WebServerSettings{
		Events: evPS,
	}

	go Webserver(s)

	for {
		select {}
	}

}
<commit_msg>delete nonsense test<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"time"
)

func main() {
	args := os.Args[1:]
	if len(args) != 2 {
		log.Fatal("Usage: gensite <src dir> <dst dir>")
	}
	srcDir, dstDir := args[0], args[1]
	srcFile := func(elem ...string) string {
		elem = append([]string{srcDir}, elem...)
		return filepath.Join(elem...)
	}
	dstFile := func(elem ...string) string {
		elem = append([]string{dstDir}, elem...)
		return filepath.Join(elem...)
	}

	// Read site configuration.
	conf := &siteConf{}
	decodeTOML(srcFile("index.toml"), conf)
	if conf.RootURL == "" {
		log.Fatal("RootURL must be specified; needed by feed and sitemap")
	}
	if conf.Template == "" {
		log.Fatal("Template must be specified")
	}
	if conf.BaseCSS == nil {
		log.Fatal("BaseCSS must be specified")
	}

	template := readFile(srcFile(conf.Template))
	baseCSS := catInDir(srcDir, conf.BaseCSS)

	// Initialize templates. They are all initialized from the same source code,
	// plus a snippet to fix the "content" reference.
	categoryTmpl := newTemplate("category", "..", template, contentIs("category"))
	articleTmpl := newTemplate("article", "..", template, contentIs("article"))
	homepageTmpl := newTemplate("homepage", ".", template, contentIs("article"))
	feedTmpl := newTemplate("feed", ".", feedTemplText)

	// Base for the {{ . }} object used in all templates.
	base := newBaseDot(conf, baseCSS)

	// Up to conf.FeedPosts recent posts, used in the feed.
	recents := recentArticles{nil, conf.FeedPosts}
	// Last modified time of the newest post, used in the feed.
	var lastModified time.Time

	// Whether the "all" category has been requested.
	hasAllCategory := false
	// Meta of all articles, used to generate the index of the "all", if if is
	// requested.
	allArticleMetas := []articleMeta{}

	// Paths of all generated URLs, relative to the destination directory,
	// always without "index.html". Used to generate the sitemap.
	allPaths := []string{""}

	// Render a category index.
	renderCategoryIndex := func(name, prelude, css, js string, groups []group) {
		// Add category index to the sitemap, without "/index.html"
		allPaths = append(allPaths, name)
		// Create directory
		catDir := dstFile(name)
		err := os.MkdirAll(catDir, 0755)
		if err != nil {
			log.Fatal(err)
		}

		// Generate index
		cd := &categoryDot{base, name, prelude, groups, css, js}
		executeToFile(categoryTmpl, cd, filepath.Join(catDir, "index.html"))
	}

	for _, cat := range conf.Categories {
		if cat.Name == "all" {
			// The "all" category has been requested. It is a pseudo-category in
			// that it doesn't need to have any associated category
			// configuration file. We cannot render the category index now
			// because we haven't seen all articles yet. Render it later.
			hasAllCategory = true
			continue
		}

		catConf := &categoryConf{}
		decodeTOML(srcFile(cat.Name, "index.toml"), catConf)

		prelude := ""
		if catConf.Prelude != "" {
			prelude = readFile(srcFile(cat.Name, catConf.Prelude+".html"))
		}
		css := catInDir(srcFile(cat.Name), catConf.ExtraCSS)
		js := catInDir(srcFile(cat.Name), catConf.ExtraJS)
		var groups []group
		if catConf.AutoIndex {
			groups = makeGroups(catConf.Articles, catConf.Groups)
		}
		renderCategoryIndex(cat.Name, prelude, css, js, groups)

		// Generate articles
		for _, am := range catConf.Articles {
			// Add article URL to sitemap.
			p := filepath.Join(cat.Name, am.Name+".html")
			allPaths = append(allPaths, p)

			a := getArticle(article{Category: cat.Name}, am, srcFile(cat.Name))
			modTime := time.Time(a.LastModified)
			if modTime.After(lastModified) {
				lastModified = modTime
			}

			// Generate article page.
			ad := &articleDot{base, a}
			executeToFile(articleTmpl, ad, dstFile(p))

			allArticleMetas = append(allArticleMetas, a.articleMeta)
			recents.insert(a)
		}
	}

	// Generate "all category"
	if hasAllCategory {
		sort.Slice(allArticleMetas, func(i, j int) bool {
			return allArticleMetas[i].Timestamp > allArticleMetas[j].Timestamp
		})
		renderCategoryIndex("all", "", "", "", []group{{Articles: allArticleMetas}})
	}

	// Generate index page. XXX(xiaq): duplicated code with generating ordinary
	// article pages.
	a := getArticle(article{IsHomepage: true, Category: "homepage"}, conf.Index, srcDir)
	ad := &articleDot{base, a}
	executeToFile(homepageTmpl, ad, dstFile("index.html"))

	// Generate feed.
	feedArticles := recents.articles
	fd := feedDot{base, feedArticles, rfc3339Time(lastModified)}
	executeToFile(feedTmpl, fd, dstFile("feed.atom"))

	// Generate site map.
	file := openForWrite(dstFile("sitemap.txt"))
	defer file.Close()
	for _, p := range allPaths {
		fmt.Fprintf(file, "%s/%s\n", conf.RootURL, p)
	}
}

func makeGroups(articles []articleMeta, groupMetas []groupMeta) []group {
	groups := make(map[int]*group)
	for _, am := range articles {
		g := groups[am.Group]
		if g == nil {
			g = &group{}
			if 0 <= am.Group && am.Group < len(groupMetas) {
				g.groupMeta = groupMetas[am.Group]
			}
			groups[am.Group] = g
		}
		g.Articles = append(g.Articles, am)
	}
	indices := make([]int, 0, len(groups))
	for i := range groups {
		indices = append(indices, i)
	}
	sort.Ints(indices)
	sortedGroups := make([]group, len(groups))
	for i, idx := range indices {
		sortedGroups[i] = *groups[idx]
	}
	return sortedGroups
}
<commit_msg>website: Remove support for the "all" category from gensite.<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"time"
)

func main() {
	args := os.Args[1:]
	if len(args) != 2 {
		log.Fatal("Usage: gensite <src dir> <dst dir>")
	}
	srcDir, dstDir := args[0], args[1]
	srcFile := func(elem ...string) string {
		elem = append([]string{srcDir}, elem...)
		return filepath.Join(elem...)
	}
	dstFile := func(elem ...string) string {
		elem = append([]string{dstDir}, elem...)
		return filepath.Join(elem...)
	}

	// Read site configuration.
	conf := &siteConf{}
	decodeTOML(srcFile("index.toml"), conf)
	if conf.RootURL == "" {
		log.Fatal("RootURL must be specified; needed by feed and sitemap")
	}
	if conf.Template == "" {
		log.Fatal("Template must be specified")
	}
	if conf.BaseCSS == nil {
		log.Fatal("BaseCSS must be specified")
	}

	template := readFile(srcFile(conf.Template))
	baseCSS := catInDir(srcDir, conf.BaseCSS)

	// Initialize templates. They are all initialized from the same source code,
	// plus a snippet to fix the "content" reference.
	categoryTmpl := newTemplate("category", "..", template, contentIs("category"))
	articleTmpl := newTemplate("article", "..", template, contentIs("article"))
	homepageTmpl := newTemplate("homepage", ".", template, contentIs("article"))
	feedTmpl := newTemplate("feed", ".", feedTemplText)

	// Base for the {{ . }} object used in all templates.
	base := newBaseDot(conf, baseCSS)

	// Up to conf.FeedPosts recent posts, used in the feed.
	recents := recentArticles{nil, conf.FeedPosts}
	// Last modified time of the newest post, used in the feed.
	var lastModified time.Time

	// Paths of all generated URLs, relative to the destination directory,
	// always without "index.html". Used to generate the sitemap.
	allPaths := []string{""}

	// Render a category index.
	renderCategoryIndex := func(name, prelude, css, js string, groups []group) {
		// Add category index to the sitemap, without "/index.html"
		allPaths = append(allPaths, name)
		// Create directory
		catDir := dstFile(name)
		err := os.MkdirAll(catDir, 0755)
		if err != nil {
			log.Fatal(err)
		}

		// Generate index
		cd := &categoryDot{base, name, prelude, groups, css, js}
		executeToFile(categoryTmpl, cd, filepath.Join(catDir, "index.html"))
	}

	for _, cat := range conf.Categories {
		catConf := &categoryConf{}
		decodeTOML(srcFile(cat.Name, "index.toml"), catConf)

		prelude := ""
		if catConf.Prelude != "" {
			prelude = readFile(srcFile(cat.Name, catConf.Prelude+".html"))
		}
		css := catInDir(srcFile(cat.Name), catConf.ExtraCSS)
		js := catInDir(srcFile(cat.Name), catConf.ExtraJS)
		var groups []group
		if catConf.AutoIndex {
			groups = makeGroups(catConf.Articles, catConf.Groups)
		}
		renderCategoryIndex(cat.Name, prelude, css, js, groups)

		// Generate articles
		for _, am := range catConf.Articles {
			// Add article URL to sitemap.
			p := filepath.Join(cat.Name, am.Name+".html")
			allPaths = append(allPaths, p)

			a := getArticle(article{Category: cat.Name}, am, srcFile(cat.Name))
			modTime := time.Time(a.LastModified)
			if modTime.After(lastModified) {
				lastModified = modTime
			}

			// Generate article page.
			ad := &articleDot{base, a}
			executeToFile(articleTmpl, ad, dstFile(p))

			recents.insert(a)
		}
	}

	// Generate index page. XXX(xiaq): duplicated code with generating ordinary
	// article pages.
	a := getArticle(article{IsHomepage: true, Category: "homepage"}, conf.Index, srcDir)
	ad := &articleDot{base, a}
	executeToFile(homepageTmpl, ad, dstFile("index.html"))

	// Generate feed.
	feedArticles := recents.articles
	fd := feedDot{base, feedArticles, rfc3339Time(lastModified)}
	executeToFile(feedTmpl, fd, dstFile("feed.atom"))

	// Generate site map.
	file := openForWrite(dstFile("sitemap.txt"))
	defer file.Close()
	for _, p := range allPaths {
		fmt.Fprintf(file, "%s/%s\n", conf.RootURL, p)
	}
}

func makeGroups(articles []articleMeta, groupMetas []groupMeta) []group {
	groups := make(map[int]*group)
	for _, am := range articles {
		g := groups[am.Group]
		if g == nil {
			g = &group{}
			if 0 <= am.Group && am.Group < len(groupMetas) {
				g.groupMeta = groupMetas[am.Group]
			}
			groups[am.Group] = g
		}
		g.Articles = append(g.Articles, am)
	}
	indices := make([]int, 0, len(groups))
	for i := range groups {
		indices = append(indices, i)
	}
	sort.Ints(indices)
	sortedGroups := make([]group, len(groups))
	for i, idx := range indices {
		sortedGroups[i] = *groups[idx]
	}
	return sortedGroups
}
<|endoftext|>


<commit_before>// Copyright 2015 Red Hat Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package doc

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/cpuguy83/go-md2man/md2man"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

// GenManTree will generate a man page for this command and all descendants
// in the directory given. The header may be nil. This function may not work
// correctly if your command names have `-` in them. If you have `cmd` with two
// subcmds, `sub` and `sub-third`, and `sub` has a subcommand called `third`
// it is undefined which help output will be in the file `cmd-sub-third.1`.
func GenManTree(cmd *cobra.Command, header *GenManHeader, dir string) error {
	return GenManTreeFromOpts(cmd, GenManTreeOptions{
		Header:           header,
		Path:             dir,
		CommandSeparator: "-",
	})
}

// GenManTreeFromOpts generates a man page for the command and all descendants.
// The pages are written to the opts.Path directory.
func GenManTreeFromOpts(cmd *cobra.Command, opts GenManTreeOptions) error {
	header := opts.Header
	if header == nil {
		header = &GenManHeader{}
	}
	for _, c := range cmd.Commands() {
		if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
			continue
		}
		if err := GenManTreeFromOpts(c, opts); err != nil {
			return err
		}
	}
	section := "1"
	if header.Section != "" {
		section = header.Section
	}

	separator := "_"
	if opts.CommandSeparator != "" {
		separator = opts.CommandSeparator
	}
	basename := strings.Replace(cmd.CommandPath(), " ", separator, -1)
	filename := filepath.Join(opts.Path, basename+"."+section)
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	headerCopy := *header
	return GenMan(cmd, &headerCopy, f)
}

// GenManTreeOptions is the options for generating the man pages.
// Used only in GenManTreeFromOpts.
type GenManTreeOptions struct {
	Header           *GenManHeader
	Path             string
	CommandSeparator string
}

// GenManHeader is a lot like the .TH header at the start of man pages. These
// include the title, section, date, source, and manual. We will use the
// current time if Date is unset and will use "Auto generated by spf13/cobra"
// if the Source is unset.
type GenManHeader struct {
	Title   string
	Section string
	Date    *time.Time
	date    string
	Source  string
	Manual  string
}

// GenMan will generate a man page for the given command and write it to
// w. The header argument may be nil, however obviously w may not.
func GenMan(cmd *cobra.Command, header *GenManHeader, w io.Writer) error {
	if header == nil {
		header = &GenManHeader{}
	}
	fillHeader(header, cmd.CommandPath())

	b := genMan(cmd, header)
	_, err := w.Write(md2man.Render(b))
	return err
}

func fillHeader(header *GenManHeader, name string) {
	if header.Title == "" {
		header.Title = strings.ToUpper(strings.Replace(name, " ", "\\-", -1))
	}
	if header.Section == "" {
		header.Section = "1"
	}
	if header.Date == nil {
		now := time.Now()
		header.Date = &now
	}
	header.date = (*header.Date).Format("Jan 2006")
	if header.Source == "" {
		header.Source = "Auto generated by spf13/cobra"
	}
}

func manPreamble(buf *bytes.Buffer, header *GenManHeader, cmd *cobra.Command, dashedName string) {
	description := cmd.Long
	if len(description) == 0 {
		description = cmd.Short
	}

	buf.WriteString(fmt.Sprintf(`%% %s(%s)%s
%% %s
%% %s
# NAME
`, header.Title, header.Section, header.date, header.Source, header.Manual))
	buf.WriteString(fmt.Sprintf("%s \\- %s\n\n", dashedName, cmd.Short))
	buf.WriteString("# SYNOPSIS\n")
	buf.WriteString(fmt.Sprintf("**%s**\n\n", cmd.UseLine()))
	buf.WriteString("# DESCRIPTION\n")
	buf.WriteString(description + "\n\n")
}

func manPrintFlags(buf *bytes.Buffer, flags *pflag.FlagSet) {
	flags.VisitAll(func(flag *pflag.Flag) {
		if len(flag.Deprecated) > 0 || flag.Hidden {
			return
		}
		format := ""
		if len(flag.Shorthand) > 0 && len(flag.ShorthandDeprecated) == 0 {
			format = fmt.Sprintf("**-%s**, **--%s**", flag.Shorthand, flag.Name)
		} else {
			format = fmt.Sprintf("**--%s**", flag.Name)
		}
		if len(flag.NoOptDefVal) > 0 {
			format += "["
		}
		if flag.Value.Type() == "string" {
			// put quotes on the value
			format += "=%q"
		} else {
			format += "=%s"
		}
		if len(flag.NoOptDefVal) > 0 {
			format += "]"
		}
		format += "\n\t%s\n\n"
		buf.WriteString(fmt.Sprintf(format, flag.DefValue, flag.Usage))
	})
}

func manPrintOptions(buf *bytes.Buffer, command *cobra.Command) {
	flags := command.NonInheritedFlags()
	if flags.HasAvailableFlags() {
		buf.WriteString("# OPTIONS\n")
		manPrintFlags(buf, flags)
		buf.WriteString("\n")
	}
	flags = command.InheritedFlags()
	if flags.HasAvailableFlags() {
		buf.WriteString("# OPTIONS INHERITED FROM PARENT COMMANDS\n")
		manPrintFlags(buf, flags)
		buf.WriteString("\n")
	}
}

func genMan(cmd *cobra.Command, header *GenManHeader) []byte {
	cmd.InitDefaultHelpCmd()
	cmd.InitDefaultHelpFlag()

	// something like `rootcmd-subcmd1-subcmd2`
	dashCommandName := strings.Replace(cmd.CommandPath(), " ", "-", -1)

	buf := new(bytes.Buffer)

	manPreamble(buf, header, cmd, dashCommandName)
	manPrintOptions(buf, cmd)
	if len(cmd.Example) > 0 {
		buf.WriteString("# EXAMPLE\n")
		buf.WriteString(fmt.Sprintf("```\n%s\n```\n", cmd.Example))
	}
	if hasSeeAlso(cmd) {
		buf.WriteString("# SEE ALSO\n")
		seealsos := make([]string, 0)
		if cmd.HasParent() {
			parentPath := cmd.Parent().CommandPath()
			dashParentPath := strings.Replace(parentPath, " ", "-", -1)
			seealso := fmt.Sprintf("**%s(%s)**", dashParentPath, header.Section)
			seealsos = append(seealsos, seealso)
			cmd.VisitParents(func(c *cobra.Command) {
				if c.DisableAutoGenTag {
					cmd.DisableAutoGenTag = c.DisableAutoGenTag
				}
			})
		}
		children := cmd.Commands()
		sort.Sort(byName(children))
		for _, c := range children {
			if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
				continue
			}
			seealso := fmt.Sprintf("**%s-%s(%s)**", dashCommandName, c.Name(), header.Section)
			seealsos = append(seealsos, seealso)
		}
		buf.WriteString(strings.Join(seealsos, ", ") + "\n")
	}
	if !cmd.DisableAutoGenTag {
		buf.WriteString(fmt.Sprintf("# HISTORY\n%s Auto generated by spf13/cobra\n", header.Date.Format("2-Jan-2006")))
	}
	return buf.Bytes()
}
<commit_msg>doc: obey SOURCE_DATE_EPOCH with manpage generation (#735)<commit_after>// Copyright 2015 Red Hat Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package doc

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cpuguy83/go-md2man/md2man"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

// GenManTree will generate a man page for this command and all descendants
// in the directory given. The header may be nil. This function may not work
// correctly if your command names have `-` in them. If you have `cmd` with two
// subcmds, `sub` and `sub-third`, and `sub` has a subcommand called `third`
// it is undefined which help output will be in the file `cmd-sub-third.1`.
func GenManTree(cmd *cobra.Command, header *GenManHeader, dir string) error {
	return GenManTreeFromOpts(cmd, GenManTreeOptions{
		Header:           header,
		Path:             dir,
		CommandSeparator: "-",
	})
}

// GenManTreeFromOpts generates a man page for the command and all descendants.
// The pages are written to the opts.Path directory.
func GenManTreeFromOpts(cmd *cobra.Command, opts GenManTreeOptions) error {
	header := opts.Header
	if header == nil {
		header = &GenManHeader{}
	}
	for _, c := range cmd.Commands() {
		if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
			continue
		}
		if err := GenManTreeFromOpts(c, opts); err != nil {
			return err
		}
	}
	section := "1"
	if header.Section != "" {
		section = header.Section
	}

	separator := "_"
	if opts.CommandSeparator != "" {
		separator = opts.CommandSeparator
	}
	basename := strings.Replace(cmd.CommandPath(), " ", separator, -1)
	filename := filepath.Join(opts.Path, basename+"."+section)
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	headerCopy := *header
	return GenMan(cmd, &headerCopy, f)
}

// GenManTreeOptions is the options for generating the man pages.
// Used only in GenManTreeFromOpts.
type GenManTreeOptions struct {
	Header           *GenManHeader
	Path             string
	CommandSeparator string
}

// GenManHeader is a lot like the .TH header at the start of man pages. These
// include the title, section, date, source, and manual. We will use the
// current time if Date is unset and will use "Auto generated by spf13/cobra"
// if the Source is unset.
type GenManHeader struct {
	Title   string
	Section string
	Date    *time.Time
	date    string
	Source  string
	Manual  string
}

// GenMan will generate a man page for the given command and write it to
// w. The header argument may be nil, however obviously w may not.
func GenMan(cmd *cobra.Command, header *GenManHeader, w io.Writer) error {
	if header == nil {
		header = &GenManHeader{}
	}
	if err := fillHeader(header, cmd.CommandPath()); err != nil {
		return err
	}

	b := genMan(cmd, header)
	_, err := w.Write(md2man.Render(b))
	return err
}

func fillHeader(header *GenManHeader, name string) error {
	if header.Title == "" {
		header.Title = strings.ToUpper(strings.Replace(name, " ", "\\-", -1))
	}
	if header.Section == "" {
		header.Section = "1"
	}
	if header.Date == nil {
		now := time.Now()
		if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" {
			unixEpoch, err := strconv.ParseInt(epoch, 10, 64)
			if err != nil {
				return fmt.Errorf("invalid SOURCE_DATE_EPOCH: %v", err)
			}
			now = time.Unix(unixEpoch, 0)
		}
		header.Date = &now
	}
	header.date = (*header.Date).Format("Jan 2006")
	if header.Source == "" {
		header.Source = "Auto generated by spf13/cobra"
	}
	return nil
}

func manPreamble(buf *bytes.Buffer, header *GenManHeader, cmd *cobra.Command, dashedName string) {
	description := cmd.Long
	if len(description) == 0 {
		description = cmd.Short
	}

	buf.WriteString(fmt.Sprintf(`%% %s(%s)%s
%% %s
%% %s
# NAME
`, header.Title, header.Section, header.date, header.Source, header.Manual))
	buf.WriteString(fmt.Sprintf("%s \\- %s\n\n", dashedName, cmd.Short))
	buf.WriteString("# SYNOPSIS\n")
	buf.WriteString(fmt.Sprintf("**%s**\n\n", cmd.UseLine()))
	buf.WriteString("# DESCRIPTION\n")
	buf.WriteString(description + "\n\n")
}

func manPrintFlags(buf *bytes.Buffer, flags *pflag.FlagSet) {
	flags.VisitAll(func(flag *pflag.Flag) {
		if len(flag.Deprecated) > 0 || flag.Hidden {
			return
		}
		format := ""
		if len(flag.Shorthand) > 0 && len(flag.ShorthandDeprecated) == 0 {
			format = fmt.Sprintf("**-%s**, **--%s**", flag.Shorthand, flag.Name)
		} else {
			format = fmt.Sprintf("**--%s**", flag.Name)
		}
		if len(flag.NoOptDefVal) > 0 {
			format += "["
		}
		if flag.Value.Type() == "string" {
			// put quotes on the value
			format += "=%q"
		} else {
			format += "=%s"
		}
		if len(flag.NoOptDefVal) > 0 {
			format += "]"
		}
		format += "\n\t%s\n\n"
		buf.WriteString(fmt.Sprintf(format, flag.DefValue, flag.Usage))
	})
}

func manPrintOptions(buf *bytes.Buffer, command *cobra.Command) {
	flags := command.NonInheritedFlags()
	if flags.HasAvailableFlags() {
		buf.WriteString("# OPTIONS\n")
		manPrintFlags(buf, flags)
		buf.WriteString("\n")
	}
	flags = command.InheritedFlags()
	if flags.HasAvailableFlags() {
		buf.WriteString("# OPTIONS INHERITED FROM PARENT COMMANDS\n")
		manPrintFlags(buf, flags)
		buf.WriteString("\n")
	}
}

func genMan(cmd *cobra.Command, header *GenManHeader) []byte {
	cmd.InitDefaultHelpCmd()
	cmd.InitDefaultHelpFlag()

	// something like `rootcmd-subcmd1-subcmd2`
	dashCommandName := strings.Replace(cmd.CommandPath(), " ", "-", -1)

	buf := new(bytes.Buffer)

	manPreamble(buf, header, cmd, dashCommandName)
	manPrintOptions(buf, cmd)
	if len(cmd.Example) > 0 {
		buf.WriteString("# EXAMPLE\n")
		buf.WriteString(fmt.Sprintf("```\n%s\n```\n", cmd.Example))
	}
	if hasSeeAlso(cmd) {
		buf.WriteString("# SEE ALSO\n")
		seealsos := make([]string, 0)
		if cmd.HasParent() {
			parentPath := cmd.Parent().CommandPath()
			dashParentPath := strings.Replace(parentPath, " ", "-", -1)
			seealso := fmt.Sprintf("**%s(%s)**", dashParentPath, header.Section)
			seealsos = append(seealsos, seealso)
			cmd.VisitParents(func(c *cobra.Command) {
				if c.DisableAutoGenTag {
					cmd.DisableAutoGenTag = c.DisableAutoGenTag
				}
			})
		}
		children := cmd.Commands()
		sort.Sort(byName(children))
		for _, c := range children {
			if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
				continue
			}
			seealso := fmt.Sprintf("**%s-%s(%s)**", dashCommandName, c.Name(), header.Section)
			seealsos = append(seealsos, seealso)
		}
		buf.WriteString(strings.Join(seealsos, ", ") + "\n")
	}
	if !cmd.DisableAutoGenTag {
		buf.WriteString(fmt.Sprintf("# HISTORY\n%s Auto generated by spf13/cobra\n", header.Date.Format("2-Jan-2006")))
	}
	return buf.Bytes()
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fs"
	"github.com/hanwen/go-fuse/v2/fuse"
	"io"
	"strings"
	"syscall"
)

/** Rename a file
 *
 * If the target exists it should be atomically replaced. If
 * the target's inode's lookup count is non-zero, the file
 * system is expected to postpone any removal of the inode
 * until the lookup count reaches zero (see description of the
 * forget function).
 *
 * If this request is answered with an error code of ENOSYS, this is
 * treated as a permanent failure with error code EINVAL, i.e. all
 * future bmap requests will fail with EINVAL without being
 * send to the filesystem process.
 *
 * *flags* may be `RENAME_EXCHANGE` or `RENAME_NOREPLACE`. If
 * RENAME_NOREPLACE is specified, the filesystem must not
 * overwrite *newname* if it exists and return an error
 * instead. If `RENAME_EXCHANGE` is specified, the filesystem
 * must atomically exchange the two files, i.e. both must
 * exist and neither may be deleted.
 *
 * Valid replies:
 *   fuse_reply_err
 *
 * @param req request handle
 * @param parent inode number of the old parent directory
 * @param name old name
 * @param newparent inode number of the new parent directory
 * @param newname new name
 */
/*
renameat2()
       renameat2() has an additional flags argument.  A renameat2() call
       with a zero flags argument is equivalent to renameat().

       The flags argument is a bit mask consisting of zero or more of
       the following flags:

       RENAME_EXCHANGE
              Atomically exchange oldpath and newpath.  Both pathnames
              must exist but may be of different types (e.g., one could
              be a non-empty directory and the other a symbolic link).

       RENAME_NOREPLACE
              Don't overwrite newpath of the rename.  Return an error if
              newpath already exists.

              RENAME_NOREPLACE can't be employed together with
              RENAME_EXCHANGE.

              RENAME_NOREPLACE requires support from the underlying
              filesystem.  Support for various filesystems was added as
              follows:

              *  ext4 (Linux 3.15);

              *  btrfs, tmpfs, and cifs (Linux 3.17);

              *  xfs (Linux 4.0);

              *  Support for many other filesystems was added in Linux
                 4.9, including ext2, minix, reiserfs, jfs, vfat, and
                 bpf.

       RENAME_WHITEOUT (since Linux 3.18)
              This operation makes sense only for overlay/union
              filesystem implementations.

              Specifying RENAME_WHITEOUT creates a "whiteout" object at
              the source of the rename at the same time as performing
              the rename.  The whole operation is atomic, so that if the
              rename succeeds then the whiteout will also have been
              created.

              A "whiteout" is an object that has special meaning in
              union/overlay filesystem constructs.  In these constructs,
              multiple layers exist and only the top one is ever
              modified.  A whiteout on an upper layer will effectively
              hide a matching file in the lower layer, making it appear
              as if the file didn't exist.

              When a file that exists on the lower layer is renamed, the
              file is first copied up (if not already on the upper
              layer) and then renamed on the upper, read-write layer.
              At the same time, the source file needs to be "whiteouted"
              (so that the version of the source file in the lower layer
              is rendered invisible).  The whole operation needs to be
              done atomically.

              When not part of a union/overlay, the whiteout appears as
              a character device with a {0,0} device number.  (Note that
              other union/overlay implementations may employ different
              methods for storing whiteout entries; specifically, BSD
              union mount employs a separate inode type, DT_WHT, which,
              while supported by some filesystems available in Linux,
              such as CODA and XFS, is ignored by the kernel's whiteout
              support code, as of Linux 4.19, at least.)

              RENAME_WHITEOUT requires the same privileges as creating a
              device node (i.e., the CAP_MKNOD capability).

              RENAME_WHITEOUT can't be employed together with
              RENAME_EXCHANGE.

              RENAME_WHITEOUT requires support from the underlying
              filesystem.  Among the filesystems that support it are
              tmpfs (since Linux 3.18), ext4 (since Linux 3.18), XFS
              (since Linux 4.1), f2fs (since Linux 4.2), btrfs (since
              Linux 4.7), and ubifs (since Linux 4.9).
*/
const (
	RenameEmptyFlag = 0
	RenameNoReplace = 1
	RenameExchange  = fs.RENAME_EXCHANGE
	RenameWhiteout  = 3
)

func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string, newName string) (code fuse.Status) {
	if wfs.IsOverQuota {
		return fuse.Status(syscall.ENOSPC)
	}

	if s := checkName(newName); s != fuse.OK {
		return s
	}

	switch in.Flags {
	case RenameEmptyFlag:
	case RenameNoReplace:
	case RenameExchange:
	case RenameWhiteout:
		return fuse.ENOTSUP
	default:
		return fuse.EINVAL
	}

	oldDir, code := wfs.inodeToPath.GetPath(in.NodeId)
	if code != fuse.OK {
		return
	}
	oldPath := oldDir.Child(oldName)
	newDir, code := wfs.inodeToPath.GetPath(in.Newdir)
	if code != fuse.OK {
		return
	}
	newPath := newDir.Child(newName)

	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)

	// update remote filer
	err := wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		request := &filer_pb.StreamRenameEntryRequest{
			OldDirectory: string(oldDir),
			OldName:      oldName,
			NewDirectory: string(newDir),
			NewName:      newName,
			Signatures:   []int32{wfs.signature},
		}

		stream, err := client.StreamRenameEntry(ctx, request)
		if err != nil {
			code = fuse.EIO
			return fmt.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err)
		}

		for {
			resp, recvErr := stream.Recv()
			if recvErr != nil {
				if recvErr == io.EOF {
					break
				} else {
					if strings.Contains(recvErr.Error(), "not empty") {
						code = fuse.Status(syscall.ENOTEMPTY)
					} else if strings.Contains(recvErr.Error(), "not directory") {
						code = fuse.ENOTDIR
					}
					return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr)
				}
			}

			if err = wfs.handleRenameResponse(ctx, resp); err != nil {
				glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
				return err
			}

		}

		return nil

	})
	if err != nil {
		glog.V(0).Infof("Link: %v", err)
		return
	}

	return fuse.OK

}

func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
	// comes from filer StreamRenameEntry, can only be create or delete entry

	glog.V(4).Infof("dir Rename %+v", resp.EventNotification)

	if resp.EventNotification.NewEntry != nil {
		// with new entry, the old entry name also exists. This is the first step to create new entry
		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry, false); err != nil {
			return err
		}

		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
		oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name

		oldPath := oldParent.Child(oldName)
		newPath := newParent.Child(newName)

		sourceInode, targetInode := wfs.inodeToPath.MovePath(oldPath, newPath)
		if sourceInode != 0 {
			if fh := wfs.fhmap.inode2fh[sourceInode]; foundFh && fh.entry != nil {
				fh.entry.Name = newName
			}
			wfs.fuseServer.InodeNotify(sourceInode, 0, -1)
		}
		// invalidate attr and data
		if targetInode != 0 {
			wfs.fuseServer.InodeNotify(targetInode, 0, -1)
		}

	} else if resp.EventNotification.OldEntry != nil {
		// without new entry, only old entry name exists. This is the second step to delete old entry
		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil, resp.EventNotification.DeleteChunks); err != nil {
			return err
		}
	}

	return nil

}
<commit_msg>comments<commit_after>package mount

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fs"
	"github.com/hanwen/go-fuse/v2/fuse"
	"io"
	"strings"
	"syscall"
)

/** Rename a file
 *
 * If the target exists it should be atomically replaced. If
 * the target's inode's lookup count is non-zero, the file
 * system is expected to postpone any removal of the inode
 * until the lookup count reaches zero (see description of the
 * forget function).
 *
 * If this request is answered with an error code of ENOSYS, this is
 * treated as a permanent failure with error code EINVAL, i.e. all
 * future bmap requests will fail with EINVAL without being
 * send to the filesystem process.
 *
 * *flags* may be `RENAME_EXCHANGE` or `RENAME_NOREPLACE`. If
 * RENAME_NOREPLACE is specified, the filesystem must not
 * overwrite *newname* if it exists and return an error
 * instead. If `RENAME_EXCHANGE` is specified, the filesystem
 * must atomically exchange the two files, i.e. both must
 * exist and neither may be deleted.
 *
 * Valid replies:
 *   fuse_reply_err
 *
 * @param req request handle
 * @param parent inode number of the old parent directory
 * @param name old name
 * @param newparent inode number of the new parent directory
 * @param newname new name
 */
/*
renameat2()
       renameat2() has an additional flags argument.  A renameat2() call
       with a zero flags argument is equivalent to renameat().

       The flags argument is a bit mask consisting of zero or more of
       the following flags:

       RENAME_EXCHANGE
              Atomically exchange oldpath and newpath.  Both pathnames
              must exist but may be of different types (e.g., one could
              be a non-empty directory and the other a symbolic link).

       RENAME_NOREPLACE
              Don't overwrite newpath of the rename.  Return an error if
              newpath already exists.

              RENAME_NOREPLACE can't be employed together with
              RENAME_EXCHANGE.

              RENAME_NOREPLACE requires support from the underlying
              filesystem.  Support for various filesystems was added as
              follows:

              *  ext4 (Linux 3.15);

              *  btrfs, tmpfs, and cifs (Linux 3.17);

              *  xfs (Linux 4.0);

              *  Support for many other filesystems was added in Linux
                 4.9, including ext2, minix, reiserfs, jfs, vfat, and
                 bpf.

       RENAME_WHITEOUT (since Linux 3.18)
              This operation makes sense only for overlay/union
              filesystem implementations.

              Specifying RENAME_WHITEOUT creates a "whiteout" object at
              the source of the rename at the same time as performing
              the rename.  The whole operation is atomic, so that if the
              rename succeeds then the whiteout will also have been
              created.

              A "whiteout" is an object that has special meaning in
              union/overlay filesystem constructs.  In these constructs,
              multiple layers exist and only the top one is ever
              modified.  A whiteout on an upper layer will effectively
              hide a matching file in the lower layer, making it appear
              as if the file didn't exist.

              When a file that exists on the lower layer is renamed, the
              file is first copied up (if not already on the upper
              layer) and then renamed on the upper, read-write layer.
              At the same time, the source file needs to be "whiteouted"
              (so that the version of the source file in the lower layer
              is rendered invisible).  The whole operation needs to be
              done atomically.

              When not part of a union/overlay, the whiteout appears as
              a character device with a {0,0} device number.  (Note that
              other union/overlay implementations may employ different
              methods for storing whiteout entries; specifically, BSD
              union mount employs a separate inode type, DT_WHT, which,
              while supported by some filesystems available in Linux,
              such as CODA and XFS, is ignored by the kernel's whiteout
              support code, as of Linux 4.19, at least.)

              RENAME_WHITEOUT requires the same privileges as creating a
              device node (i.e., the CAP_MKNOD capability).

              RENAME_WHITEOUT can't be employed together with
              RENAME_EXCHANGE.

              RENAME_WHITEOUT requires support from the underlying
              filesystem.  Among the filesystems that support it are
              tmpfs (since Linux 3.18), ext4 (since Linux 3.18), XFS
              (since Linux 4.1), f2fs (since Linux 4.2), btrfs (since
              Linux 4.7), and ubifs (since Linux 4.9).
*/
const (
	RenameEmptyFlag = 0
	RenameNoReplace = 1
	RenameExchange  = fs.RENAME_EXCHANGE
	RenameWhiteout  = 3
)

func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string, newName string) (code fuse.Status) {
	if wfs.IsOverQuota {
		return fuse.Status(syscall.ENOSPC)
	}

	if s := checkName(newName); s != fuse.OK {
		return s
	}

	switch in.Flags {
	case RenameEmptyFlag:
	case RenameNoReplace:
	case RenameExchange:
	case RenameWhiteout:
		return fuse.ENOTSUP
	default:
		return fuse.EINVAL
	}

	oldDir, code := wfs.inodeToPath.GetPath(in.NodeId)
	if code != fuse.OK {
		return
	}
	oldPath := oldDir.Child(oldName)
	newDir, code := wfs.inodeToPath.GetPath(in.Newdir)
	if code != fuse.OK {
		return
	}
	newPath := newDir.Child(newName)

	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)

	// update remote filer
	err := wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		request := &filer_pb.StreamRenameEntryRequest{
			OldDirectory: string(oldDir),
			OldName:      oldName,
			NewDirectory: string(newDir),
			NewName:      newName,
			Signatures:   []int32{wfs.signature},
		}

		stream, err := client.StreamRenameEntry(ctx, request)
		if err != nil {
			code = fuse.EIO
			return fmt.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err)
		}

		for {
			resp, recvErr := stream.Recv()
			if recvErr != nil {
				if recvErr == io.EOF {
					break
				} else {
					if strings.Contains(recvErr.Error(), "not empty") {
						code = fuse.Status(syscall.ENOTEMPTY)
					} else if strings.Contains(recvErr.Error(), "not directory") {
						code = fuse.ENOTDIR
					}
					return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr)
				}
			}

			if err = wfs.handleRenameResponse(ctx, resp); err != nil {
				glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
				return err
			}

		}

		return nil

	})
	if err != nil {
		glog.V(0).Infof("Link: %v", err)
		return
	}

	return fuse.OK

}

func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
	// comes from filer StreamRenameEntry, can only be create or delete entry

	glog.V(4).Infof("dir Rename %+v", resp.EventNotification)

	if resp.EventNotification.NewEntry != nil {
		// with new entry, the old entry name also exists. This is the first step to create new entry
		newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry)
		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry, false); err != nil {
			return err
		}

		oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath)
		oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name

		oldPath := oldParent.Child(oldName)
		newPath := newParent.Child(newName)

		sourceInode, targetInode := wfs.inodeToPath.MovePath(oldPath, newPath)
		if sourceInode != 0 {
			if fh := wfs.fhmap.inode2fh[sourceInode]; foundFh && fh.entry != nil {
				fh.entry.Name = newName
			}
			// invalidate attr and data
			wfs.fuseServer.InodeNotify(sourceInode, 0, -1)
		}
		if targetInode != 0 {
			// invalidate attr and data
			wfs.fuseServer.InodeNotify(targetInode, 0, -1)
		}

	} else if resp.EventNotification.OldEntry != nil {
		// without new entry, only old entry name exists. This is the second step to delete old entry
		if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil, resp.EventNotification.DeleteChunks); err != nil {
			return err
		}
	}

	return nil

}
<|endoftext|>