<commit_before>package lane

import (
	"fmt"
	"sync"
)

// PQType represents a priority queue ordering kind (see MAXPQ and MINPQ)
type PQType int

const (
	MAXPQ PQType = iota
	MINPQ
)

type item struct {
	value    interface{}
	priority int
}

// PQueue is a heap priority queue implementation. It can be
// whether max or min ordered and is safe
// for concurrent read-write operations.
type PQueue struct {
	sync.RWMutex
	items      []*item
	elemsCount int
	comparator func(int, int) bool
}

func newItem(value interface{}, priority int) *item {
	return &item{
		value:    value,
		priority: priority,
	}
}

func (i *item) String() string {
	return fmt.Sprintf("<item value:%s priority:%d>", i.value, i.priority)
}

// NewPQueue creates a new priority queue with the provided pqtype
// ordering type
func NewPQueue(pqType PQType) *PQueue {
	var cmp func(int, int) bool

	if pqType == MAXPQ {
		cmp = max
	} else {
		cmp = min
	}

	items := make([]*item, 1)
	items[0] = nil // Heap queue first element should always be nil

	return &PQueue{
		items:      items,
		elemsCount: 0,
		comparator: cmp,
	}
}

// Size returns the elements present in the priority queue count
func (pq *PQueue) Size() int {
	return pq.elemsCount
}

// Push the value item into the priority queue with provided priority.
func (pq *PQueue) Push(value interface{}, priority int) {
	item := newItem(value, priority)

	pq.Lock()
	pq.items = append(pq.items, item)
	pq.elemsCount += 1
	pq.swim(pq.Size())
	pq.Unlock()
}

// Pop and returns the highest/lowest priority item (depending on whether
// you're using a MINPQ or MAXPQ) from the priority queue
func (pq *PQueue) Pop() (interface{}, int) {
	if pq.Size() < 1 {
		return nil, 0
	}

	pq.Lock()

	var max *item = pq.items[1]

	pq.exch(1, pq.Size())
	pq.items = pq.items[0:pq.Size()]
	pq.elemsCount -= 1
	pq.sink(1)

	pq.Unlock()

	return max.value, max.priority
}

// Head returns the highest/lowest priority item (depending on whether
// you're using a MINPQ or MAXPQ) from the priority queue
func (pq *PQueue) Head() (interface{}, int) {
	if pq.Size() < 1 {
		return nil, 0
	}

	pq.RLock()
	headValue := pq.items[1].value
	headPriority := pq.items[1].priority
	pq.RUnlock()

	return headValue, headPriority
}

func max(i, j int) bool {
	return i < j
}

func min(i, j int) bool {
	return i > j
}

func (pq *PQueue) less(i, j int) bool {
	return pq.comparator(pq.items[i].priority, pq.items[j].priority)
}

func (pq *PQueue) exch(i, j int) {
	var tmpItem *item = pq.items[i]

	pq.items[i] = pq.items[j]
	pq.items[j] = tmpItem
}

func (pq *PQueue) swim(k int) {
	for k > 1 && pq.less(k/2, k) {
		pq.exch(k/2, k)
	}

	k = k / 2
}

func (pq *PQueue) sink(k int) {
	for 2*k <= pq.Size() {
		var j int = 2 * k

		if j < pq.Size() && pq.less(j, j+1) {
			j++
		}

		if !pq.less(k, j) {
			break
		}

		pq.exch(k, j)
		k = j
	}
}
<commit_msg>Updated priority queue documentation<commit_after>package lane

import (
	"fmt"
	"sync"
)

// PQType represents a priority queue ordering kind (see MAXPQ and MINPQ)
type PQType int

const (
	MAXPQ PQType = iota
	MINPQ
)

type item struct {
	value    interface{}
	priority int
}

// PQueue is a heap priority queue data structure implementation.
// It can be whether max or min ordered and it is synchronized
// and is safe for concurrent operations.
type PQueue struct {
	sync.RWMutex
	items      []*item
	elemsCount int
	comparator func(int, int) bool
}

func newItem(value interface{}, priority int) *item {
	return &item{
		value:    value,
		priority: priority,
	}
}

func (i *item) String() string {
	return fmt.Sprintf("<item value:%s priority:%d>", i.value, i.priority)
}

// NewPQueue creates a new priority queue with the provided pqtype
// ordering type
func NewPQueue(pqType PQType) *PQueue {
	var cmp func(int, int) bool

	if pqType == MAXPQ {
		cmp = max
	} else {
		cmp = min
	}

	items := make([]*item, 1)
	items[0] = nil // Heap queue first element should always be nil

	return &PQueue{
		items:      items,
		elemsCount: 0,
		comparator: cmp,
	}
}

// Push the value item into the priority queue with provided priority.
func (pq *PQueue) Push(value interface{}, priority int) {
	item := newItem(value, priority)

	pq.Lock()
	pq.items = append(pq.items, item)
	pq.elemsCount += 1
	pq.swim(pq.Size())
	pq.Unlock()
}

// Pop and returns the highest/lowest priority item (depending on whether
// you're using a MINPQ or MAXPQ) from the priority queue
func (pq *PQueue) Pop() (interface{}, int) {
	if pq.Size() < 1 {
		return nil, 0
	}

	pq.Lock()

	var max *item = pq.items[1]

	pq.exch(1, pq.Size())
	pq.items = pq.items[0:pq.Size()]
	pq.elemsCount -= 1
	pq.sink(1)

	pq.Unlock()

	return max.value, max.priority
}

// Head returns the highest/lowest priority item (depending on whether
// you're using a MINPQ or MAXPQ) from the priority queue
func (pq *PQueue) Head() (interface{}, int) {
	if pq.Size() < 1 {
		return nil, 0
	}

	pq.RLock()
	headValue := pq.items[1].value
	headPriority := pq.items[1].priority
	pq.RUnlock()

	return headValue, headPriority
}

// Size returns the elements present in the priority queue count
func (pq *PQueue) Size() int {
	return pq.elemsCount
}

func max(i, j int) bool {
	return i < j
}

func min(i, j int) bool {
	return i > j
}

func (pq *PQueue) less(i, j int) bool {
	return pq.comparator(pq.items[i].priority, pq.items[j].priority)
}

func (pq *PQueue) exch(i, j int) {
	var tmpItem *item = pq.items[i]

	pq.items[i] = pq.items[j]
	pq.items[j] = tmpItem
}

func (pq *PQueue) swim(k int) {
	for k > 1 && pq.less(k/2, k) {
		pq.exch(k/2, k)
	}

	k = k / 2
}

func (pq *PQueue) sink(k int) {
	for 2*k <= pq.Size() {
		var j int = 2 * k

		if j < pq.Size() && pq.less(j, j+1) {
			j++
		}

		if !pq.less(k, j) {
			break
		}

		pq.exch(k, j)
		k = j
	}
}
<|endoftext|>


<commit_before>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)
	r, err := api.Res("probes").Id(id, &p).Get()
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
//	fmt.Printf("r: %#v\np: %#v\n", r, p)
	return
}

type Probes []Probe

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p *ProbesList, err error) {
	log.Printf("GetProbes: opts=%+v", opts)
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)

	var plist *interface{}

	r, err := api.Res("probes", plist).Get()
	if err != nil {
		err = fmt.Errorf("%v - r:%v\n", err, r)
		return
	}
	fmt.Printf("r: %#v\nplist: %#v\n", r, plist)
	p = plist
	return
}
<commit_msg>Remove debug printf.<commit_after>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)
	r, err := api.Res("probes").Id(id, &p).Get()
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
	return
}

type Probes []Probe

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p *ProbesList, err error) {
	log.Printf("GetProbes: opts=%+v", opts)
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)

	var plist *interface{}

	r, err := api.Res("probes", plist).Get()
	if err != nil {
		err = fmt.Errorf("%v - r:%v\n", err, r)
		return
	}
	fmt.Printf("r: %#v\nplist: %#v\n", r, plist)
	p = plist
	return
}
<|endoftext|>


<commit_before>package p11kit

import (
	"bytes"
	"context"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"
	"time"
)

const (
	p11KitClientPath    = "/usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-client.so"
	p11KitEnvServerAddr = "P11_KIT_SERVER_ADDRESS"
	p11KitEnvServerPID  = "P11_KIT_SERVER_PID"
)

func testRequiresP11Tools(t *testing.T) {
	//	t.Skip("skipping e2e tests")
	if _, err := exec.LookPath("pkcs11-tool"); err != nil {
		t.Skip("pkcs11-tool not available, skipping test")
	}
	if _, err := os.Stat(p11KitClientPath); err != nil {
		t.Skip("p11-kit-client.so not available, skipping test")
	}
}

func newListener(t *testing.T) (net.Listener, string) {
	t.Helper()
	socketPath := filepath.Join(t.TempDir(), "p11kit.sock")
	l, err := net.Listen("unix", socketPath)
	if err != nil {
		t.Fatalf("Listening for unix socket: %v", err)
	}
	t.Cleanup(func() {
		if err := l.Close(); err != nil {
			t.Errorf("Closing unix socket: %v", err)
		}
	})
	return l, socketPath
}

type testServer struct{}

func (t *testServer) server() *Server {
	hwVersion := Version{0x01, 0x01}
	fwVersion := Version{0x02, 0x02}
	return &Server{
		Manufacturer: "test",
		Library:      "test_lib",
		LibraryVersion: Version{
			Major: 0x00,
			Minor: 0x01,
		},
		Slots: []Slot{
			{
				ID:              0x01,
				Label:           "slot-0x01",
				Manufacturer:    "test_man",
				Model:           "test_model",
				Serial:          "serial-0x01",
				HardwareVersion: hwVersion,
				FirmwareVersion: fwVersion,
			},
			{
				ID:              0x02,
				Label:           "slot-0x02",
				Manufacturer:    "test_man",
				Model:           "test_model",
				Serial:          "serial-0x02",
				HardwareVersion: hwVersion,
				FirmwareVersion: fwVersion,
			},
		},
	}
}

func TestPKCS11Tool(t *testing.T) {
	testRequiresP11Tools(t)

	tests := []struct {
		name string
		args []string
	}{
		{"ListSlots", []string{"--list-slots"}},
		{"ListTokenSlots", []string{"--list-token-slots"}},
		{"ListObjects", []string{"--list-objects"}},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			l, path := newListener(t)

			ts := testServer{}
			h := ts.server()

			errCh := make(chan error)
			go func() {
				conn, err := l.Accept()
				if err != nil {
					errCh <- err
					return
				}
				conn.SetDeadline(time.Now().Add(time.Second * 10))
				errCh <- h.Handle(conn)
			}()

			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
			defer cancel()

			var stdout, stderr bytes.Buffer
			cmd := exec.CommandContext(ctx, "pkcs11-tool",
				append([]string{
					"--verbose",
					"--module", p11KitClientPath,
				}, test.args...)...)
			cmd.Env = append(os.Environ(),
				"P11_KIT_DEBUG=all",
				p11KitEnvServerPID+"="+strconv.Itoa(os.Getpid()),
				p11KitEnvServerAddr+"=unix:path="+path,
			)
			cmd.Stdout = &stdout
			cmd.Stderr = &stderr

			if err := cmd.Run(); err != nil {
				t.Errorf("command failed: %v: %s", err, &stderr)
			} else {
				t.Logf("%s", &stdout)
			}
			if err := <-errCh; err != nil {
				t.Errorf("handle error: %v", err)
			}
		})
	}
}
<commit_msg>*: improve test experience if pkcs11-tool fails before executing<commit_after>package p11kit

import (
	"bytes"
	"context"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"
	"time"
)

const (
	p11KitClientPath    = "/usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-client.so"
	p11KitEnvServerAddr = "P11_KIT_SERVER_ADDRESS"
	p11KitEnvServerPID  = "P11_KIT_SERVER_PID"
)

func testRequiresP11Tools(t *testing.T) {
	//	t.Skip("skipping e2e tests")
	if _, err := exec.LookPath("pkcs11-tool"); err != nil {
		t.Skip("pkcs11-tool not available, skipping test")
	}
	if _, err := os.Stat(p11KitClientPath); err != nil {
		t.Skip("p11-kit-client.so not available, skipping test")
	}
}

func newListener(t *testing.T) (net.Listener, string) {
	t.Helper()
	socketPath := filepath.Join(t.TempDir(), "p11kit.sock")
	l, err := net.Listen("unix", socketPath)
	if err != nil {
		t.Fatalf("Listening for unix socket: %v", err)
	}
	t.Cleanup(func() {
		if err := l.Close(); err != nil {
			t.Errorf("Closing unix socket: %v", err)
		}
	})
	return l, socketPath
}

type testServer struct{}

func (t *testServer) server() *Server {
	hwVersion := Version{0x01, 0x01}
	fwVersion := Version{0x02, 0x02}
	return &Server{
		Manufacturer: "test",
		Library:      "test_lib",
		LibraryVersion: Version{
			Major: 0x00,
			Minor: 0x01,
		},
		Slots: []Slot{
			{
				ID:              0x01,
				Label:           "slot-0x01",
				Manufacturer:    "test_man",
				Model:           "test_model",
				Serial:          "serial-0x01",
				HardwareVersion: hwVersion,
				FirmwareVersion: fwVersion,
			},
			{
				ID:              0x02,
				Label:           "slot-0x02",
				Manufacturer:    "test_man",
				Model:           "test_model",
				Serial:          "serial-0x02",
				HardwareVersion: hwVersion,
				FirmwareVersion: fwVersion,
			},
		},
	}
}

func TestPKCS11Tool(t *testing.T) {
	testRequiresP11Tools(t)

	tests := []struct {
		name string
		args []string
	}{
		{"ListSlots", []string{"--list-slots"}},
		{"ListTokenSlots", []string{"--list-token-slots"}},
		{"ListObjects", []string{"--list-objects"}},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			l, path := newListener(t)

			ts := testServer{}
			h := ts.server()

			errCh := make(chan error)
			go func() {
				done := make(chan struct{})
				defer close(done)
				go func() {
					select {
					case <-done:
					case <-time.After(time.Second * 10):
						l.Close()
					}
				}()
				conn, err := l.Accept()
				if err != nil {
					errCh <- err
					return
				}
				conn.SetDeadline(time.Now().Add(time.Second * 10))
				errCh <- h.Handle(conn)
			}()

			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
			defer cancel()

			var stdout, stderr bytes.Buffer
			cmd := exec.CommandContext(ctx, "pkcs11-tool",
				append([]string{
					"--verbose",
					"--module", p11KitClientPath,
				}, test.args...)...)
			cmd.Env = append(os.Environ(),
				"P11_KIT_DEBUG=all",
				p11KitEnvServerPID+"="+strconv.Itoa(os.Getpid()),
				p11KitEnvServerAddr+"=unix:path="+path,
			)
			cmd.Stdout = &stdout
			cmd.Stderr = &stderr

			if err := cmd.Run(); err != nil {
				t.Errorf("command failed: %v\nstderr=%s\nstdout=%s", err, &stderr, &stdout)
			} else {
				t.Logf("%s", &stdout)
			}
			if err := <-errCh; err != nil {
				t.Errorf("handle error: %v", err)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bufio"
	"compress/flate"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
)

// from archive/zip struct.go

const (
	fileHeaderSignature      = 0x04034b50
	directoryHeaderSignature = 0x02014b50
	directoryEndSignature    = 0x06054b50
	directory64LocSignature  = 0x07064b50
	directory64EndSignature  = 0x06064b50
	dataDescriptorSignature  = 0x08074b50 // de-facto standard; required by OS X Finder
	fileHeaderLen            = 30         // + filename + extra
	directoryHeaderLen       = 46         // + filename + extra + comment
	directoryEndLen          = 22         // + comment
	dataDescriptorLen        = 16         // four uint32: descriptor signature, crc32, compressed size, size
	dataDescriptor64Len      = 24         // descriptor with 8 byte sizes
	directory64LocLen        = 20         //
	directory64EndLen        = 56         // + extra

	// Constants for the first byte in CreatorVersion
	creatorFAT    = 0
	creatorUnix   = 3
	creatorNTFS   = 11
	creatorVFAT   = 14
	creatorMacOSX = 19

	// version numbers
	zipVersion20 = 20 // 2.0
	zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
)

type readBuf []byte

func (b *readBuf) uint16() uint16 {
	v := binary.LittleEndian.Uint16(*b)
	*b = (*b)[2:]
	return v
}

func (b *readBuf) uint32() uint32 {
	v := binary.LittleEndian.Uint32(*b)
	*b = (*b)[4:]
	return v
}

func (b *readBuf) uint64() uint64 {
	v := binary.LittleEndian.Uint64(*b)
	*b = (*b)[8:]
	return v
}

func main() {
	debug := flag.Bool("debug", false, "print debug info")

	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatal("usage: ", os.Args[0], " {zip-file}")
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal("open ", err)
	}

	defer f.Close()

	r := bufio.NewReader(f)

	for {
		fmt.Println()

		var fh [fileHeaderLen]byte

		if _, err := io.ReadFull(r, fh[:]); err != nil {
			log.Fatal("file header ", err)
		}

		if *debug {
			fmt.Println(hex.Dump(fh[:]))
		}

		b := readBuf(fh[:])
		magic := b.uint32()
		if magic != fileHeaderSignature {
			log.Fatal("invalid file header signature ", magic)
		}

		fmt.Printf("magic   %08x\n", magic)
		fmt.Printf("version %04x\n", b.uint16())
		flags := b.uint16()
		comp := b.uint16()
		fmt.Printf("flags   %04x\n", flags)
		fmt.Printf("comp    %04x\n", comp)
		fmt.Printf("time    %04x\n", b.uint16())
		fmt.Printf("date    %04x\n", b.uint16())
		fmt.Printf("crc32   %08x\n", b.uint32())
		clen := b.uint32()
		ulen := b.uint32()
		fmt.Printf("compressed size   %d\n", clen)
		fmt.Printf("uncompressed size %d\n", ulen)
		flen := b.uint16()
		elen := b.uint16()
		fmt.Printf("filename length   %d\n", flen)
		fmt.Printf("extra length      %d\n", elen)

		fn := make([]byte, flen)
		if _, err := io.ReadFull(r, fn); err != nil {
			log.Fatal("read file name ", err)
		}

		if elen > 0 {
			if _, err := io.CopyN(ioutil.Discard, r, int64(elen)); err != nil {
				log.Fatal("read extra ", err)
			}
		}

		fmt.Println()
		fmt.Println("filename", string(fn))

		if comp == zip.Deflate {
			dec := flate.NewReader(r)
			n, err := io.Copy(ioutil.Discard, dec)
			fmt.Println("decoded", n, "bytes")
			if err != nil {
				log.Fatal("decode file ", err)
			} else {
				dec.Close()
			}
		} else if ulen > 0 {
			n, err := io.CopyN(ioutil.Discard, r, int64(ulen))
			fmt.Println("read", n, "bytes")
			if err != nil {
				log.Fatal("read file ", err)
			}
		} else {
			log.Fatal("missing lenght")
		}

		if (flags & 0x08) != 0 {
			// data descriptor
			var dd [dataDescriptorLen]byte

			if _, err := io.ReadFull(r, dd[:]); err != nil {
				log.Fatal("data descriptor", err)
			}

			b := readBuf(dd[:])
			fmt.Println()

			magic := b.uint32()
			if magic != dataDescriptorSignature {
				log.Fatal("invalid data descriptor signature ", magic)
			}

			fmt.Printf("magic   %08x\n", magic)
			fmt.Printf("crc32   %08x\n", b.uint32())
			fmt.Printf("compressed size   %d\n", b.uint32())
			fmt.Printf("uncompressed size %d\n", b.uint32())
		}
	}
}
<commit_msg>some cleanup in preparation of adding "unzip" modes (view or extract)<commit_after>package main

import (
	"archive/zip"
	"bufio"
	"compress/flate"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
)

// from archive/zip struct.go

const (
	fileHeaderSignature     = 0x04034b50
	dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder
	fileHeaderLen           = 30         // + filename + extra
	dataDescriptorLen       = 16         // four uint32: descriptor signature, crc32, compressed size, size
	dataDescriptor64Len     = 24         // descriptor with 8 byte sizes

	// version numbers
	zipVersion20 = 20 // 2.0
	zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
)

type readBuf []byte

func (b *readBuf) uint16() uint16 {
	v := binary.LittleEndian.Uint16(*b)
	*b = (*b)[2:]
	return v
}

func (b *readBuf) uint32() uint32 {
	v := binary.LittleEndian.Uint32(*b)
	*b = (*b)[4:]
	return v
}

func (b *readBuf) uint64() uint64 {
	v := binary.LittleEndian.Uint64(*b)
	*b = (*b)[8:]
	return v
}

func main() {
	debug := flag.Bool("debug", false, "print debug info")
	//view := flag.Bool("v", false, "view list")

	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatal("usage: ", os.Args[0], " {zip-file}")
	}

	zipfile := flag.Arg(0)

	f, err := os.Open(zipfile)
	if err != nil {
		log.Fatal("open ", err)
	}

	defer f.Close()

	r := bufio.NewReader(f)

	for {
		fmt.Println()

		var fh [fileHeaderLen]byte

		if _, err := io.ReadFull(r, fh[:]); err != nil {
			log.Fatal("file header ", err)
		}

		if *debug {
			fmt.Println(hex.Dump(fh[:]))
		}

		b := readBuf(fh[:])
		magic := b.uint32()
		version := b.uint16()
		flags := b.uint16()
		comp := b.uint16()
		ctime := b.uint16()
		cdate := b.uint16()
		crc32 := b.uint32()
		clen := b.uint32()
		ulen := b.uint32()
		flen := b.uint16()
		elen := b.uint16()

		if magic != fileHeaderSignature {
			log.Fatal("invalid file header signature ", magic)
		}

		if *debug {
			fmt.Printf("magic   %08x\n", magic)
			fmt.Printf("version %04x\n", version)
			fmt.Printf("flags   %04x\n", flags)
			fmt.Printf("comp    %04x\n", comp)
			fmt.Printf("time    %04x\n", ctime)
			fmt.Printf("date    %04x\n", cdate)
			fmt.Printf("crc32   %08x\n", crc32)
			fmt.Printf("compressed size   %d\n", clen)
			fmt.Printf("uncompressed size %d\n", ulen)
			fmt.Printf("filename length   %d\n", flen)
			fmt.Printf("extra length      %d\n", elen)
		}

		fn := make([]byte, flen)
		if _, err := io.ReadFull(r, fn); err != nil {
			log.Fatal("read file name ", err)
		}

		if *debug {
			fmt.Println()
			fmt.Println("filename", string(fn))
		}

		if elen > 0 {
			if _, err := io.CopyN(ioutil.Discard, r, int64(elen)); err != nil {
				log.Fatal("read extra ", err)
			}
		}

		switch comp {
		case zip.Deflate:
			dec := flate.NewReader(r)
			n, err := io.Copy(ioutil.Discard, dec)
			fmt.Println("decoded", n, "bytes")
			if err != nil {
				log.Fatal("decode file ", err)
			} else {
				dec.Close()
			}

		case zip.Store:
			if ulen > 0 {
				n, err := io.CopyN(ioutil.Discard, r, int64(ulen))
				fmt.Println("read", n, "bytes")
				if err != nil {
					log.Fatal("read file ", err)
				}
			} else {
				log.Fatal("missing lenght")
			}

		default:
			log.Fatal("unsupported compression mode ", comp)
		}

		if (flags & 0x08) != 0 {
			// data descriptor
			var dd [dataDescriptorLen]byte

			if _, err := io.ReadFull(r, dd[:]); err != nil {
				log.Fatal("data descriptor", err)
			}

			b := readBuf(dd[:])
			fmt.Println()

			magic := b.uint32()
			crc32 = b.uint32()
			clen = b.uint32()
			ulen = b.uint32()

			if magic != dataDescriptorSignature {
				log.Fatal("invalid data descriptor signature ", magic)
			}

			if *debug {
				fmt.Printf("magic   %08x\n", magic)
				fmt.Printf("crc32   %08x\n", crc32)
				fmt.Printf("compressed size   %d\n", clen)
				fmt.Printf("uncompressed size %d\n", ulen)
			}
		}
	}
}
<|endoftext|>


<commit_before>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
	"log"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	key, ok := HasAPIKey()
	api := gopencils.Api(apiEndpoint, nil)

	// Add at least one option, the APIkey if present
	var opts map[string]string

	if ok {
		opts["key"] = key
	}

	r, err := api.Res("probes").Id(id, &p).Get(opts)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
	return
}

// probeList is our main answer
type probeList struct {
	Count    int
	Next     string
	Previous string
	Results  []Probe
}

// fetch the given resource
func fetchOneProbePage(api *gopencils.Resource, opts map[string]string) (raw *probeList, err error) {
	r, err := api.Res("probes", &raw).Get(opts)
	if err != nil {
		log.Printf("err: %v", err)
		err = fmt.Errorf("%v - r:%v\n", err, r)
	}
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
	key, ok := HasAPIKey()
	api := gopencils.Api(apiEndpoint, nil)

	// Add APIKey if set
	if ok {
		opts["key"] = key
	}

	// First call
	rawlist, err := fetchOneProbePage(api, opts)

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty probe list")
	}

	var res []Probe

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneProbePage(api, opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	p = res
	return
}
<commit_msg>Properly initialize opts.<commit_after>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
	"log"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	key, ok := HasAPIKey()
	api := gopencils.Api(apiEndpoint, nil)

	// Add at least one option, the APIkey if present
	var opts = make(map[string]string)

	if ok {
		opts["key"] = key
	}

	r, err := api.Res("probes").Id(id, &p).Get(opts)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
	return
}

// probeList is our main answer
type probeList struct {
	Count    int
	Next     string
	Previous string
	Results  []Probe
}

// fetch the given resource
func fetchOneProbePage(api *gopencils.Resource, opts map[string]string) (raw *probeList, err error) {
	r, err := api.Res("probes", &raw).Get(opts)
	if err != nil {
		log.Printf("err: %v", err)
		err = fmt.Errorf("%v - r:%v\n", err, r)
	}
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
	key, ok := HasAPIKey()
	api := gopencils.Api(apiEndpoint, nil)

	// Add APIKey if set
	if ok {
		opts["key"] = key
	}

	// First call
	rawlist, err := fetchOneProbePage(api, opts)

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty probe list")
	}

	var res []Probe

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneProbePage(api, opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	p = res
	return
}
<|endoftext|>


<commit_before>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
	"log"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	key, ok := HasAPIKey()
	api := gopencils.Api(apiEndpoint, nil)

	// Add at least one option, the APIkey if present
	var opts = make(map[string]string)

	if ok {
		opts["key"] = key
	}

	r, err := api.Res("probes").Id(id, &p).Get(opts)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
	return
}

// probeList is our main answer
type probeList struct {
	Count    int
	Next     string
	Previous string
	Results  []Probe
}

// fetch the given resource
func fetchOneProbePage(api *gopencils.Resource, opts map[string]string) (raw *probeList, err error) {
	r, err := api.Res("probes", &raw).Get(opts)
	if err != nil {
		log.Printf("err: %v", err)
		err = fmt.Errorf("%v - r:%v\n", err, r)
	}
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
	key, ok := HasAPIKey()
	api := gopencils.Api(apiEndpoint, nil)

	// Add APIKey if set
	if ok {
		opts["key"] = key
	}

	// First call
	rawlist, err := fetchOneProbePage(api, opts)

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty probe list")
	}

	var res []Probe

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneProbePage(api, opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	p = res
	return
}
<commit_msg>Use Sendgrid rest package instead of gopencils.<commit_after>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"encoding/json"
	"fmt"
	"github.com/sendgrid/rest"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	probeEP := apiEndpoint + "/probes"

	key, ok := HasAPIKey()

	// Add at least one option, the APIkey if present
	hdrs := make(map[string]string)
	opts := make(map[string]string)

	if ok {
		opts["key"] = key
	}

	req := rest.Request{
		BaseURL:     probeEP + fmt.Sprintf("/%d", id),
		Method:      rest.Get,
		Headers:     hdrs,
		QueryParams: opts,
	}

	//log.Printf("req: %#v", req)
	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}

	p = &Probe{}
	err = json.Unmarshal([]byte(r.Body), p)
	//log.Printf("json: %#v\n", p)
	return
}

// probeList is our main answer
type probeList struct {
	Count    int
	Next     string
	Previous string
	Results  []Probe
}

// fetch the given resource
func fetchOneProbePage(probeEP string, opts map[string]string) (raw *probeList, err error) {
	hdrs := make(map[string]string)
	req := rest.Request{
		BaseURL:     probeEP,
		Method:      rest.Get,
		Headers:     hdrs,
		QueryParams: opts,
	}

	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}

	raw = &probeList{}
	err = json.Unmarshal([]byte(r.Body), raw)
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
	probeEP := apiEndpoint + "/probes"

	key, ok := HasAPIKey()

	// Add APIKey if set
	if ok {
		opts["key"] = key
	}

	// First call
	rawlist, err := fetchOneProbePage(probeEP, opts)

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty probe list")
	}

	var res []Probe

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneProbePage(probeEP, opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	p = res
	return
}
<|endoftext|>


<commit_before>package mstools

import (
	"log"
	"math"
	"os"
)

//Round function to round, with precision 2
func Round(input float64) float32 {
	var round float64
	pow := math.Pow(10, float64(2))
	digit := pow * input
	round = math.Ceil(digit)
	return float32(round / pow)
}

//errCheck log and die on error
func errCheck(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

// does file exist?
func fExists(fName string) (bool, error) {
	if _, err := os.Stat(fName); err == nil {
		return true, err
	}
	return false, err
}
<commit_msg>1rs builded<commit_after>package mstools

import (
	"log"
	"math"
	"os"
)

//Round function to round, with precision 2
func Round(input float64) float32 {
	var round float64
	pow := math.Pow(10, float64(2))
	digit := pow * input
	round = math.Ceil(digit)
	return float32(round / pow)
}

//errCheck log and die on error
func errCheck(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

// does file exist?
func fExists(fName string) (exist bool , err error) {
	exist = false
	if _, err := os.Stat(fName); err == nil {
		exist = true
	}
	return
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

var (
	Config        = NewConfig()
	defaultRemote = "origin"
)

// FetchPruneConfig collects together the config options that control fetching and pruning
type FetchPruneConfig struct {
	// The number of days prior to current date for which (local) refs other than HEAD
	// will be fetched with --recent (default 7, 0 = only fetch HEAD)
	FetchRecentRefsDays int
	// Makes the FetchRecentRefsDays option apply to remote refs from fetch source as well (default true)
	FetchRecentRefsIncludeRemotes bool
	// number of days prior to latest commit on a ref that we'll fetch previous
	// LFS changes too (default 0 = only fetch at ref)
	FetchRecentCommitsDays int
	// Whether to always fetch recent even without --recent
	FetchRecentAlways bool
	// Number of days added to FetchRecent*; data outside combined window will be
	// deleted when prune is run. (default 3)
	PruneOffsetDays int
}

type Configuration struct {
	CurrentRemote         string
	httpClient            *HttpClient
	redirectingHttpClient *http.Client
	envVars               map[string]string
	isTracingHttp         bool
	isLoggingStats        bool

	loading           sync.Mutex // guards initialization of gitConfig and remotes
	gitConfig         map[string]string
	origConfig        map[string]string
	remotes           []string
	extensions        map[string]Extension
	fetchIncludePaths []string
	fetchExcludePaths []string
	fetchPruneConfig  *FetchPruneConfig
}

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		envVars:       make(map[string]string),
	}
	c.isTracingHttp = c.GetenvBool("GIT_CURL_VERBOSE", false)
	c.isLoggingStats = c.GetenvBool("GIT_LOG_STATS", false)
	return c
}

func (c *Configuration) Getenv(key string) string {
	if i, ok := c.envVars[key]; ok {
		return i
	}

	v := os.Getenv(key)
	c.envVars[key] = v
	return v
}

func (c *Configuration) Setenv(key, value string) error {
	// Check see if we have this in our cache, if so update it
	if _, ok := c.envVars[key]; ok {
		c.envVars[key] = value
	}

	// Now set in process
	return os.Setenv(key, value)
}

// GetenvBool parses a boolean environment variable and returns the result as a bool.
// If the environment variable is unset, empty, or if the parsing fails,
// the value of def (default) is returned instead.
func (c *Configuration) GetenvBool(key string, def bool) bool {
	s := c.Getenv(key)
	if len(s) == 0 {
		return def
	}

	b, err := parseConfigBool(s)
	if err != nil {
		return def
	}
	return b
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return NewEndpoint(url)
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	value, ok := c.GitConfig("lfs.batch")
	if !ok || len(value) == 0 {
		return true
	}

	useBatch, err := parseConfigBool(value)
	if err != nil {
		return false
	}

	return useBatch
}

// PrivateAccess will retrieve the access value and return true if
// the value is set to private. When a repo is marked as having private
// access, the http requests for the batch api will fetch the credentials
// before running, otherwise the request will run without credentials.
func (c *Configuration) PrivateAccess() bool {
	return c.Access() != "none"
}

// Access returns the access auth type.
func (c *Configuration) Access() string {
	return c.EndpointAccess(c.Endpoint())
}

// SetAccess will set the private access flag in .git/config.
func (c *Configuration) SetAccess(authType string) {
	c.SetEndpointAccess(c.Endpoint(), authType)
}

func (c *Configuration) EndpointAccess(e Endpoint) string {
	key := fmt.Sprintf("lfs.%s.access", e.Url)

	if v, ok := c.GitConfig(key); ok && len(v) > 0 {
		lower := strings.ToLower(v)
		if lower == "private" {
			return "basic"
		}
		return lower
	}

	return "none"
}

func (c *Configuration) SetEndpointAccess(e Endpoint, authType string) {
	tracerx.Printf("setting repository access to %s", authType)
	key := fmt.Sprintf("lfs.%s.access", e.Url)

	// Modify the config cache because it's checked again in this process
	// without being reloaded.
	switch authType {
	case "", "none":
		git.Config.UnsetLocalKey("", key)

		c.loading.Lock()
		delete(c.gitConfig, strings.ToLower(key))
		c.loading.Unlock()
	default:
		git.Config.SetLocal("", key, authType)

		c.loading.Lock()
		c.gitConfig[strings.ToLower(key)] = authType
		c.loading.Unlock()
	}
}

func (c *Configuration) FetchIncludePaths() []string {
	c.loadGitConfig()
	return c.fetchIncludePaths
}
func (c *Configuration) FetchExcludePaths() []string {
	c.loadGitConfig()
	return c.fetchExcludePaths
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}
	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return NewEndpoint(url)
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		return NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) Extensions() map[string]Extension {
	c.loadGitConfig()
	return c.extensions
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) AllGitConfig() map[string]string {
	c.loadGitConfig()
	return c.gitConfig
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) FetchPruneConfig() *FetchPruneConfig {
	if c.fetchPruneConfig == nil {
		c.fetchPruneConfig = &FetchPruneConfig{
			FetchRecentRefsDays:           7,
			FetchRecentRefsIncludeRemotes: true,
			FetchRecentCommitsDays:        0,
			PruneOffsetDays:               3,
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentRefsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentremoterefs"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = b
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentcommitsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentCommitsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentalways"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentAlways = b
			}
		}
		if v, ok := c.GitConfig("lfs.pruneoffsetdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.PruneOffsetDays = n
			}
		}

	}
	return c.fetchPruneConfig
}

func parseConfigBool(str string) (bool, error) {
	switch strings.ToLower(str) {
	case "true", "1", "on", "yes", "t":
		return true, nil
	case "false", "0", "off", "no", "f":
		return false, nil
	}
	return false, fmt.Errorf("Unable to parse %q as a boolean", str)
}

func (c *Configuration) loadGitConfig() bool {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return false
	}

	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)
	c.extensions = make(map[string]Extension)

	var output string
	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	configFile := filepath.Join(LocalWorkingDir, ".gitconfig")
	fileOutput, err := git.Config.ListFromFile(configFile)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file: %s", err))
	}

	localConfig := filepath.Join(LocalGitDir, "config")
	localOutput, err := git.Config.ListFromFile(localConfig)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file %s", err))
	}

	output = fileOutput + "\n" + listOutput + "\n" + localOutput

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := strings.ToLower(pieces[0])
		value := pieces[1]
		c.gitConfig[key] = value

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		} else if len(keyParts) == 4 && keyParts[0] == "lfs" && keyParts[1] == "extension" {
			name := keyParts[2]
			ext := c.extensions[name]
			switch keyParts[3] {
			case "clean":
				ext.Clean = value
			case "smudge":
				ext.Smudge = value
			case "priority":
				p, err := strconv.Atoi(value)
				if err == nil && p >= 0 {
					ext.Priority = p
				}
			}
			ext.Name = name
			c.extensions[name] = ext
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchinclude" {
			for _, inc := range strings.Split(value, ",") {
				inc = strings.TrimSpace(inc)
				c.fetchIncludePaths = append(c.fetchIncludePaths, inc)
			}
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchexclude" {
			for _, ex := range strings.Split(value, ",") {
				ex = strings.TrimSpace(ex)
				c.fetchExcludePaths = append(c.fetchExcludePaths, ex)
			}
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}

	return true
}
<commit_msg>アー アアアア アーアー<commit_after>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

var (
	Config        = NewConfig()
	defaultRemote = "origin"
)

// FetchPruneConfig collects together the config options that control fetching and pruning
type FetchPruneConfig struct {
	// The number of days prior to current date for which (local) refs other than HEAD
	// will be fetched with --recent (default 7, 0 = only fetch HEAD)
	FetchRecentRefsDays int
	// Makes the FetchRecentRefsDays option apply to remote refs from fetch source as well (default true)
	FetchRecentRefsIncludeRemotes bool
	// number of days prior to latest commit on a ref that we'll fetch previous
	// LFS changes too (default 0 = only fetch at ref)
	FetchRecentCommitsDays int
	// Whether to always fetch recent even without --recent
	FetchRecentAlways bool
	// Number of days added to FetchRecent*; data outside combined window will be
	// deleted when prune is run. (default 3)
	PruneOffsetDays int
}

type Configuration struct {
	CurrentRemote         string
	httpClient            *HttpClient
	redirectingHttpClient *http.Client
	envVars               map[string]string
	isTracingHttp         bool
	isLoggingStats        bool

	loading           sync.Mutex // guards initialization of gitConfig and remotes
	gitConfig         map[string]string
	origConfig        map[string]string
	remotes           []string
	extensions        map[string]Extension
	fetchIncludePaths []string
	fetchExcludePaths []string
	fetchPruneConfig  *FetchPruneConfig
}

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		envVars:       make(map[string]string),
	}
	c.isTracingHttp = c.GetenvBool("GIT_CURL_VERBOSE", false)
	c.isLoggingStats = c.GetenvBool("GIT_LOG_STATS", false)
	return c
}

func (c *Configuration) Getenv(key string) string {
	if i, ok := c.envVars[key]; ok {
		return i
	}

	v := os.Getenv(key)
	c.envVars[key] = v
	return v
}

func (c *Configuration) Setenv(key, value string) error {
	// Check see if we have this in our cache, if so update it
	if _, ok := c.envVars[key]; ok {
		c.envVars[key] = value
	}

	// Now set in process
	return os.Setenv(key, value)
}

// GetenvBool parses a boolean environment variable and returns the result as a bool.
// If the environment variable is unset, empty, or if the parsing fails,
// the value of def (default) is returned instead.
func (c *Configuration) GetenvBool(key string, def bool) bool {
	s := c.Getenv(key)
	if len(s) == 0 {
		return def
	}

	b, err := parseConfigBool(s)
	if err != nil {
		return def
	}
	return b
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return NewEndpoint(url)
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	value, ok := c.GitConfig("lfs.batch")
	if !ok || len(value) == 0 {
		return true
	}

	useBatch, err := parseConfigBool(value)
	if err != nil {
		return false
	}

	return useBatch
}

// PrivateAccess will retrieve the access value and return true if
// the value is set to private. When a repo is marked as having private
// access, the http requests for the batch api will fetch the credentials
// before running, otherwise the request will run without credentials.
func (c *Configuration) PrivateAccess() bool {
	return c.Access() != "none"
}

// Access returns the access auth type.
func (c *Configuration) Access() string {
	return c.EndpointAccess(c.Endpoint())
}

// SetAccess will set the private access flag in .git/config.
func (c *Configuration) SetAccess(authType string) {
	c.SetEndpointAccess(c.Endpoint(), authType)
}

func (c *Configuration) EndpointAccess(e Endpoint) string {
	key := fmt.Sprintf("lfs.%s.access", e.Url)
	if v, ok := c.GitConfig(key); ok && len(v) > 0 {
		lower := strings.ToLower(v)
		if lower == "private" {
			return "basic"
		}
		return lower
	}
	return "none"
}

func (c *Configuration) SetEndpointAccess(e Endpoint, authType string) {
	tracerx.Printf("setting repository access to %s", authType)
	key := fmt.Sprintf("lfs.%s.access", e.Url)

	// Modify the config cache because it's checked again in this process
	// without being reloaded.
	switch authType {
	case "", "none":
		git.Config.UnsetLocalKey("", key)

		c.loading.Lock()
		delete(c.gitConfig, strings.ToLower(key))
		c.loading.Unlock()
	default:
		git.Config.SetLocal("", key, authType)

		c.loading.Lock()
		c.gitConfig[strings.ToLower(key)] = authType
		c.loading.Unlock()
	}
}

func (c *Configuration) FetchIncludePaths() []string {
	c.loadGitConfig()
	return c.fetchIncludePaths
}
func (c *Configuration) FetchExcludePaths() []string {
	c.loadGitConfig()
	return c.fetchExcludePaths
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}
	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return NewEndpoint(url)
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		return NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) Extensions() map[string]Extension {
	c.loadGitConfig()
	return c.extensions
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) AllGitConfig() map[string]string {
	c.loadGitConfig()
	return c.gitConfig
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) FetchPruneConfig() *FetchPruneConfig {
	if c.fetchPruneConfig == nil {
		c.fetchPruneConfig = &FetchPruneConfig{
			FetchRecentRefsDays:           7,
			FetchRecentRefsIncludeRemotes: true,
			FetchRecentCommitsDays:        0,
			PruneOffsetDays:               3,
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentRefsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentremoterefs"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = b
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentcommitsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentCommitsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentalways"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentAlways = b
			}
		}
		if v, ok := c.GitConfig("lfs.pruneoffsetdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.PruneOffsetDays = n
			}
		}

	}
	return c.fetchPruneConfig
}

func parseConfigBool(str string) (bool, error) {
	switch strings.ToLower(str) {
	case "true", "1", "on", "yes", "t":
		return true, nil
	case "false", "0", "off", "no", "f":
		return false, nil
	}
	return false, fmt.Errorf("Unable to parse %q as a boolean", str)
}

func (c *Configuration) loadGitConfig() bool {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return false
	}

	uniqRemotes := make(map[string]bool)

	c.gitConfig = make(map[string]string)
	c.extensions = make(map[string]Extension)

	var output string
	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	configFile := filepath.Join(LocalWorkingDir, ".gitconfig")
	fileOutput, err := git.Config.ListFromFile(configFile)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file: %s", err))
	}

	localConfig := filepath.Join(LocalGitDir, "config")
	localOutput, err := git.Config.ListFromFile(localConfig)
	if err != nil {
		panic(fmt.Errorf("Error listing git config from file %s", err))
	}

	output = fileOutput + "\n" + listOutput + "\n" + localOutput

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		key := strings.ToLower(pieces[0])
		value := pieces[1]
		c.gitConfig[key] = value

		keyParts := strings.Split(key, ".")
		if len(keyParts) > 1 && keyParts[0] == "remote" {
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		} else if len(keyParts) == 4 && keyParts[0] == "lfs" && keyParts[1] == "extension" {
			name := keyParts[2]
			ext := c.extensions[name]
			switch keyParts[3] {
			case "clean":
				ext.Clean = value
			case "smudge":
				ext.Smudge = value
			case "priority":
				p, err := strconv.Atoi(value)
				if err == nil && p >= 0 {
					ext.Priority = p
				}
			}
			ext.Name = name
			c.extensions[name] = ext
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchinclude" {
			for _, inc := range strings.Split(value, ",") {
				inc = strings.TrimSpace(inc)
				c.fetchIncludePaths = append(c.fetchIncludePaths, inc)
			}
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchexclude" {
			for _, ex := range strings.Split(value, ",") {
				ex = strings.TrimSpace(ex)
				c.fetchExcludePaths = append(c.fetchExcludePaths, ex)
			}
		}
	}

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}

	return true
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"math"
	"os"
	"path"
	"sync"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
)

type Option struct {
	MountDirectory     string
	FilerAddresses     []string
	filerIndex         int
	FilerGrpcAddresses []string
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	DiskType           types.DiskType
	ChunkSizeLimit     int64
	ConcurrentWriters  int
	CacheDir           string
	CacheSizeMB        int64
	DataCenter         string
	Umask              os.FileMode

	MountUid   uint32
	MountGid   uint32
	MountMode  os.FileMode
	MountCtime time.Time
	MountMtime time.Time

	VolumeServerAccess string // how to access volume servers
	Cipher             bool   // whether encrypt data on volume server
	UidGidMapper       *meta_cache.UidGidMapper
}

var _ = fs.FS(&WFS{})
var _ = fs.FSStatfser(&WFS{})

type WFS struct {
	option *Option

	// contains all open handles, protected by handlesLock
	handlesLock sync.Mutex
	handles     map[uint64]*FileHandle

	bufPool sync.Pool

	stats statsCache

	root        fs.Node
	fsNodeCache *FsCache

	chunkCache *chunk_cache.TieredChunkCache
	metaCache  *meta_cache.MetaCache
	signature  int32

	// throttle writers
	concurrentWriters *util.LimitedConcurrentExecutor
	Server            *fs.Server
}
type statsCache struct {
	filer_pb.StatisticsResponse
	lastChecked int64 // unix time in seconds
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		option:  option,
		handles: make(map[uint64]*FileHandle),
		bufPool: sync.Pool{
			New: func() interface{} {
				return make([]byte, option.ChunkSizeLimit)
			},
		},
		signature: util.RandomInt32(),
	}
	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + option.FilerGrpcAddresses[0] + option.FilerMountRootPath + util.Version()))[0:8]
	cacheDir := path.Join(option.CacheDir, cacheUniqueId)
	if option.CacheSizeMB > 0 {
		os.MkdirAll(cacheDir, os.FileMode(0777)&^option.Umask)
		wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024)
	}

	wfs.metaCache = meta_cache.NewMetaCache(path.Join(cacheDir, "meta"), util.FullPath(option.FilerMountRootPath), option.UidGidMapper, func(filePath util.FullPath) {

		fsNode := NodeWithId(filePath.AsInode())
		if err := wfs.Server.InvalidateNodeData(fsNode); err != nil {
			glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err)
		}

		dir, name := filePath.DirAndName()
		parent := NodeWithId(util.FullPath(dir).AsInode())
		if dir == option.FilerMountRootPath {
			parent = NodeWithId(1)
		}
		if err := wfs.Server.InvalidateEntry(parent, name); err != nil {
			glog.V(4).Infof("InvalidateEntry %s : %v", filePath, err)
		}
	})
	startTime := time.Now()
	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
	grace.OnInterrupt(func() {
		wfs.metaCache.Shutdown()
	})

	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs, id: 1}
	wfs.fsNodeCache = newFsCache(wfs.root)

	if wfs.option.ConcurrentWriters > 0 {
		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
	}

	return wfs
}

func (wfs *WFS) Root() (fs.Node, error) {
	return wfs.root, nil
}

func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32, writeOnly bool) (fileHandle *FileHandle) {

	fullpath := file.fullpath()
	glog.V(4).Infof("AcquireHandle %s uid=%d gid=%d", fullpath, uid, gid)

	inodeId := file.Id()

	wfs.handlesLock.Lock()
	existingHandle, found := wfs.handles[inodeId]
	wfs.handlesLock.Unlock()
	if found && existingHandle != nil {
		existingHandle.f.isOpen++
		existingHandle.dirtyPages.SetWriteOnly(writeOnly)
		glog.V(4).Infof("Acquired Handle %s open %d", fullpath, existingHandle.f.isOpen)
		return existingHandle
	}

	entry, _ := file.maybeLoadEntry(context.Background())
	file.entry = entry
	fileHandle = newFileHandle(file, uid, gid, writeOnly)
	file.isOpen++

	wfs.handlesLock.Lock()
	wfs.handles[inodeId] = fileHandle
	wfs.handlesLock.Unlock()
	fileHandle.handle = inodeId

	glog.V(4).Infof("Acquired new Handle %s open %d", fullpath, file.isOpen)
	return
}

func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	glog.V(4).Infof("ReleaseHandle %s id %d current handles length %d", fullpath, handleId, len(wfs.handles))

	delete(wfs.handles, uint64(handleId))

	return
}

// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {

	glog.V(4).Infof("reading fs stats: %+v", req)

	if wfs.stats.lastChecked < time.Now().Unix()-20 {

		err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

			request := &filer_pb.StatisticsRequest{
				Collection:  wfs.option.Collection,
				Replication: wfs.option.Replication,
				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
				DiskType:    string(wfs.option.DiskType),
			}

			glog.V(4).Infof("reading filer stats: %+v", request)
			resp, err := client.Statistics(context.Background(), request)
			if err != nil {
				glog.V(0).Infof("reading filer stats %v: %v", request, err)
				return err
			}
			glog.V(4).Infof("read filer stats: %+v", resp)

			wfs.stats.TotalSize = resp.TotalSize
			wfs.stats.UsedSize = resp.UsedSize
			wfs.stats.FileCount = resp.FileCount
			wfs.stats.lastChecked = time.Now().Unix()

			return nil
		})
		if err != nil {
			glog.V(0).Infof("filer Statistics: %v", err)
			return err
		}
	}

	totalDiskSize := wfs.stats.TotalSize
	usedDiskSize := wfs.stats.UsedSize
	actualFileCount := wfs.stats.FileCount

	// Compute the total number of available blocks
	resp.Blocks = totalDiskSize / blockSize

	// Compute the number of used blocks
	numBlocks := uint64(usedDiskSize / blockSize)

	// Report the number of free and available blocks for the block size
	resp.Bfree = resp.Blocks - numBlocks
	resp.Bavail = resp.Blocks - numBlocks
	resp.Bsize = uint32(blockSize)

	// Report the total number of possible files in the file system (and those free)
	resp.Files = math.MaxInt64
	resp.Ffree = math.MaxInt64 - actualFileCount

	// Report the maximum length of a name and the minimum fragment size
	resp.Namelen = 1024
	resp.Frsize = uint32(blockSize)

	return nil
}

func (wfs *WFS) mapPbIdFromFilerToLocal(entry *filer_pb.Entry) {
	if entry.Attributes == nil {
		return
	}
	entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
}
func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) {
	if entry.Attributes == nil {
		return
	}
	entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid)
}

func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType {
	if wfs.option.VolumeServerAccess == "filerProxy" {
		return func(fileId string) (targetUrls []string, err error) {
			return []string{"http://" + wfs.getCurrentFiler() + "/?proxyChunkId=" + fileId}, nil
		}
	}
	return filer.LookupFn(wfs)
}
func (wfs *WFS) getCurrentFiler() string {
	return wfs.option.FilerAddresses[wfs.option.filerIndex]
}

type NodeWithId uint64

func (n NodeWithId) Id() uint64 {
	return uint64(n)
}
func (n NodeWithId) Attr(ctx context.Context, attr *fuse.Attr) error {
	return nil
}
<commit_msg>randomize initial filer<commit_after>package filesys

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"math"
	"math/rand"
	"os"
	"path"
	"sync"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
)

type Option struct {
	MountDirectory     string
	FilerAddresses     []string
	filerIndex         int
	FilerGrpcAddresses []string
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	DiskType           types.DiskType
	ChunkSizeLimit     int64
	ConcurrentWriters  int
	CacheDir           string
	CacheSizeMB        int64
	DataCenter         string
	Umask              os.FileMode

	MountUid   uint32
	MountGid   uint32
	MountMode  os.FileMode
	MountCtime time.Time
	MountMtime time.Time

	VolumeServerAccess string // how to access volume servers
	Cipher             bool   // whether encrypt data on volume server
	UidGidMapper       *meta_cache.UidGidMapper
}

var _ = fs.FS(&WFS{})
var _ = fs.FSStatfser(&WFS{})

type WFS struct {
	option *Option

	// contains all open handles, protected by handlesLock
	handlesLock sync.Mutex
	handles     map[uint64]*FileHandle

	bufPool sync.Pool

	stats statsCache

	root        fs.Node
	fsNodeCache *FsCache

	chunkCache *chunk_cache.TieredChunkCache
	metaCache  *meta_cache.MetaCache
	signature  int32

	// throttle writers
	concurrentWriters *util.LimitedConcurrentExecutor
	Server            *fs.Server
}
type statsCache struct {
	filer_pb.StatisticsResponse
	lastChecked int64 // unix time in seconds
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		option:  option,
		handles: make(map[uint64]*FileHandle),
		bufPool: sync.Pool{
			New: func() interface{} {
				return make([]byte, option.ChunkSizeLimit)
			},
		},
		signature: util.RandomInt32(),
	}
	wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses))
	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + option.FilerGrpcAddresses[0] + option.FilerMountRootPath + util.Version()))[0:8]
	cacheDir := path.Join(option.CacheDir, cacheUniqueId)
	if option.CacheSizeMB > 0 {
		os.MkdirAll(cacheDir, os.FileMode(0777)&^option.Umask)
		wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, cacheDir, option.CacheSizeMB, 1024*1024)
	}

	wfs.metaCache = meta_cache.NewMetaCache(path.Join(cacheDir, "meta"), util.FullPath(option.FilerMountRootPath), option.UidGidMapper, func(filePath util.FullPath) {

		fsNode := NodeWithId(filePath.AsInode())
		if err := wfs.Server.InvalidateNodeData(fsNode); err != nil {
			glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err)
		}

		dir, name := filePath.DirAndName()
		parent := NodeWithId(util.FullPath(dir).AsInode())
		if dir == option.FilerMountRootPath {
			parent = NodeWithId(1)
		}
		if err := wfs.Server.InvalidateEntry(parent, name); err != nil {
			glog.V(4).Infof("InvalidateEntry %s : %v", filePath, err)
		}
	})
	startTime := time.Now()
	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
	grace.OnInterrupt(func() {
		wfs.metaCache.Shutdown()
	})

	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs, id: 1}
	wfs.fsNodeCache = newFsCache(wfs.root)

	if wfs.option.ConcurrentWriters > 0 {
		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
	}

	return wfs
}

func (wfs *WFS) Root() (fs.Node, error) {
	return wfs.root, nil
}

func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32, writeOnly bool) (fileHandle *FileHandle) {

	fullpath := file.fullpath()
	glog.V(4).Infof("AcquireHandle %s uid=%d gid=%d", fullpath, uid, gid)

	inodeId := file.Id()

	wfs.handlesLock.Lock()
	existingHandle, found := wfs.handles[inodeId]
	wfs.handlesLock.Unlock()
	if found && existingHandle != nil {
		existingHandle.f.isOpen++
		existingHandle.dirtyPages.SetWriteOnly(writeOnly)
		glog.V(4).Infof("Acquired Handle %s open %d", fullpath, existingHandle.f.isOpen)
		return existingHandle
	}

	entry, _ := file.maybeLoadEntry(context.Background())
	file.entry = entry
	fileHandle = newFileHandle(file, uid, gid, writeOnly)
	file.isOpen++

	wfs.handlesLock.Lock()
	wfs.handles[inodeId] = fileHandle
	wfs.handlesLock.Unlock()
	fileHandle.handle = inodeId

	glog.V(4).Infof("Acquired new Handle %s open %d", fullpath, file.isOpen)
	return
}

func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
	wfs.handlesLock.Lock()
	defer wfs.handlesLock.Unlock()

	glog.V(4).Infof("ReleaseHandle %s id %d current handles length %d", fullpath, handleId, len(wfs.handles))

	delete(wfs.handles, uint64(handleId))

	return
}

// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {

	glog.V(4).Infof("reading fs stats: %+v", req)

	if wfs.stats.lastChecked < time.Now().Unix()-20 {

		err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

			request := &filer_pb.StatisticsRequest{
				Collection:  wfs.option.Collection,
				Replication: wfs.option.Replication,
				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
				DiskType:    string(wfs.option.DiskType),
			}

			glog.V(4).Infof("reading filer stats: %+v", request)
			resp, err := client.Statistics(context.Background(), request)
			if err != nil {
				glog.V(0).Infof("reading filer stats %v: %v", request, err)
				return err
			}
			glog.V(4).Infof("read filer stats: %+v", resp)

			wfs.stats.TotalSize = resp.TotalSize
			wfs.stats.UsedSize = resp.UsedSize
			wfs.stats.FileCount = resp.FileCount
			wfs.stats.lastChecked = time.Now().Unix()

			return nil
		})
		if err != nil {
			glog.V(0).Infof("filer Statistics: %v", err)
			return err
		}
	}

	totalDiskSize := wfs.stats.TotalSize
	usedDiskSize := wfs.stats.UsedSize
	actualFileCount := wfs.stats.FileCount

	// Compute the total number of available blocks
	resp.Blocks = totalDiskSize / blockSize

	// Compute the number of used blocks
	numBlocks := uint64(usedDiskSize / blockSize)

	// Report the number of free and available blocks for the block size
	resp.Bfree = resp.Blocks - numBlocks
	resp.Bavail = resp.Blocks - numBlocks
	resp.Bsize = uint32(blockSize)

	// Report the total number of possible files in the file system (and those free)
	resp.Files = math.MaxInt64
	resp.Ffree = math.MaxInt64 - actualFileCount

	// Report the maximum length of a name and the minimum fragment size
	resp.Namelen = 1024
	resp.Frsize = uint32(blockSize)

	return nil
}

func (wfs *WFS) mapPbIdFromFilerToLocal(entry *filer_pb.Entry) {
	if entry.Attributes == nil {
		return
	}
	entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
}
func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) {
	if entry.Attributes == nil {
		return
	}
	entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid)
}

func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType {
	if wfs.option.VolumeServerAccess == "filerProxy" {
		return func(fileId string) (targetUrls []string, err error) {
			return []string{"http://" + wfs.getCurrentFiler() + "/?proxyChunkId=" + fileId}, nil
		}
	}
	return filer.LookupFn(wfs)
}
func (wfs *WFS) getCurrentFiler() string {
	return wfs.option.FilerAddresses[wfs.option.filerIndex]
}

type NodeWithId uint64

func (n NodeWithId) Id() uint64 {
	return uint64(n)
}
func (n NodeWithId) Attr(ctx context.Context, attr *fuse.Attr) error {
	return nil
}
<|endoftext|>


<commit_before>package hdhomerun

import (
	"encoding/hex"
	"testing"
)

func TestPacketDump(t *testing.T) {
	println("Request: \n" + hex.Dump(discoverReq.b))
	expected := "  Type: Discover Request\n"
	expected += "Length: 12\n"
	expected += "  Tags:\n"
	expected += "             Device Type: tuner\n"
	expected += "               Device ID: *\n"
	received := discoverReq.p.dump()
	if received != expected {
		t.Errorf("Unexpected output:\nExpected:\n%s\n     Got:\n%s", expected, received)
	}
}

func TestTlvDumpDeviceType(t *testing.T) {
	tests := []struct {
		expected string
		tag      tlv
	}{
		{
			expected: "       Device Type: *",
			tag: tlv{
				tag:    TagDeviceType,
				length: 0x04,
				value:  DeviceTypeWildcard,
			},
		},
		{
			expected: "       Device Type: tuner",
			tag: tlv{
				tag:    TagDeviceType,
				length: 0x04,
				value:  DeviceTypeTuner,
			},
		},
		{
			expected: "       Device Type: storage",
			tag: tlv{
				tag:    TagDeviceType,
				length: 0x04,
				value:  DeviceTypeStorage,
			},
		},
	}
	for _, test := range tests {
		received := test.tag.dump()
		if test.expected != received {
			t.Errorf("Unexpected output:\nExpected: \"%s\"\n     Got: \"%s\"", test.expected, received)
		}
	}
}

func TestTlvDumpDeviceId(t *testing.T) {
	expected := "         Device ID: 0x01020304"
	received := discoverRpy.p.tags[1].dump()
	if received != expected {
		t.Errorf("Unexpected output:\nExpected: \"%s\"\n     Got: \"%s\"", expected, received)
	}
}
<commit_msg>Removed needless debug message<commit_after>package hdhomerun

import (
	"encoding/hex"
	"testing"
)

func TestPacketDump(t *testing.T) {
	expected := "  Type: Discover Request\n"
	expected += "Length: 12\n"
	expected += "  Tags:\n"
	expected += "             Device Type: tuner\n"
	expected += "               Device ID: *\n"
	received := discoverReq.p.dump()
	if received != expected {
		t.Errorf("Unexpected output:\nExpected:\n%s\n     Got:\n%s", expected, received)
	}
}

func TestTlvDumpDeviceType(t *testing.T) {
	tests := []struct {
		expected string
		tag      tlv
	}{
		{
			expected: "       Device Type: *",
			tag: tlv{
				tag:    TagDeviceType,
				length: 0x04,
				value:  DeviceTypeWildcard,
			},
		},
		{
			expected: "       Device Type: tuner",
			tag: tlv{
				tag:    TagDeviceType,
				length: 0x04,
				value:  DeviceTypeTuner,
			},
		},
		{
			expected: "       Device Type: storage",
			tag: tlv{
				tag:    TagDeviceType,
				length: 0x04,
				value:  DeviceTypeStorage,
			},
		},
	}
	for _, test := range tests {
		received := test.tag.dump()
		if test.expected != received {
			t.Errorf("Unexpected output:\nExpected: \"%s\"\n     Got: \"%s\"", test.expected, received)
		}
	}
}

func TestTlvDumpDeviceId(t *testing.T) {
	expected := "         Device ID: 0x01020304"
	received := discoverRpy.p.tags[1].dump()
	if received != expected {
		t.Errorf("Unexpected output:\nExpected: \"%s\"\n     Got: \"%s\"", expected, received)
	}
}
<|endoftext|>


<commit_before>package mvc

import (
	"crypto/rand"
	"database/sql"
	"database/sql/driver"
	"encoding/base32"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"strings"
	"sync/atomic"
	"time"
)

var (
	counter uint32
	randID  = make([]byte, 5)
	osPID   [2]byte
)

func init() {
	if _, err := rand.Read(randID); err != nil {
		panic("Unable to get random for SQLID generation")
	}
	pid := os.Getpid()
	osPID[0] = byte(pid >> 8)
	osPID[1] = byte(pid)
}

type SQLExecutor interface {
	Prepare(query string) (*sql.Stmt, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

type SQLID string

func (id *SQLID) Scan(value interface{}) error {
	if s, ok := value.(string); ok {
		*id = SQLID(s)
	} else if b, ok := value.([]byte); ok {
		*id = SQLID(b)
	} else {
		*id = ""
	}
	return nil
}

func (id SQLID) Value() (driver.Value, error) {
	return string(id), nil
}

func (id SQLID) ToSQLNullID() SQLNullID {
	return SQLNullID{
		sql.NullString{
			String: string(id),
			Valid:  id != "",
		},
	}
}

type SQLNullString struct {
	sql.NullString
}

func (ns SQLNullString) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.String)
	}
	return json.Marshal(nil)
}

type SQLNullID struct {
	sql.NullString
}

func (ns SQLNullID) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.String)
	}
	return json.Marshal(nil)
}

func NewSQLID() SQLID {
	var b [15]byte
	binary.BigEndian.PutUint32(b[:], uint32(time.Now().Unix()))

	b[4] = randID[0]
	b[5] = randID[1]
	b[6] = randID[2]
	b[7] = randID[3]
	b[8] = randID[4]

	b[9] = osPID[0]
	b[10] = osPID[1]

	binary.BigEndian.PutUint32(b[11:], atomic.AddUint32(&counter, 1))
	return SQLID(base32.StdEncoding.EncodeToString(b[:]))
}

func NewSQLNullID() SQLNullID {
	return SQLNullID{
		sql.NullString{
			String: "",
			Valid:  false,
		},
	}
}

func NewSQLNullString() SQLNullString {
	return SQLNullString{
		sql.NullString{
			String: "",
			Valid:  false,
		},
	}
}

func IsSQLID(id string) bool {
	return len(id) == 24
}

func SQLInsert(db SQLExecutor, table string, m interface{}) (sql.Result, error) {
	if m == nil {
		return nil, nil
	}

	var v reflect.Value

	if v = reflect.ValueOf(m); v.Type().Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if v.Type().Kind() != reflect.Struct {
		return nil, nil
	}

	var (
		t            = v.Type()
		l            = t.NumField()
		fields       = make([]string, 0, l)
		values       = make([]interface{}, 0, l)
		valuesholder = make([]string, 0, l)
		query        *sql.Stmt
		err          error
	)

	for i := 0; i < l; i++ {
		if field := t.Field(i); field.PkgPath == "" {
			fname := field.Tag.Get("db")
			if fname == "" {
				fname = strings.ToLower(field.Name)
			}
			iface := v.Field(i).Interface()
			if t, ok := iface.(time.Time); ok && t.IsZero() {
				continue
			}
			fields = append(fields, fname)
			values = append(values, iface)
			valuesholder = append(valuesholder, "?")
		}
	}
	stmt := fmt.Sprintf("INSERT INTO `%s`(`%s`) VALUES(%s)",
		table,
		strings.Join(fields, "`,`"),
		strings.Join(valuesholder, ","))

	if query, err = db.Prepare(stmt); err == nil {
		return query.Exec(values...)
	}

	return nil, err
}
<commit_msg>Added sql.Null* wrapper for proper json serialization<commit_after>package mvc

import (
	"crypto/rand"
	"database/sql"
	"database/sql/driver"
	"encoding/base32"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"strings"
	"sync/atomic"
	"time"
)

var (
	counter uint32
	randID  = make([]byte, 5)
	osPID   [2]byte
)

func init() {
	if _, err := rand.Read(randID); err != nil {
		panic("Unable to get random for SQLID generation")
	}
	pid := os.Getpid()
	osPID[0] = byte(pid >> 8)
	osPID[1] = byte(pid)
}

type SQLExecutor interface {
	Prepare(query string) (*sql.Stmt, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

type SQLID string

func (id *SQLID) Scan(value interface{}) error {
	if s, ok := value.(string); ok {
		*id = SQLID(s)
	} else if b, ok := value.([]byte); ok {
		*id = SQLID(b)
	} else {
		*id = ""
	}
	return nil
}

func (id SQLID) Value() (driver.Value, error) {
	return string(id), nil
}

func (id SQLID) ToSQLNullID() SQLNullID {
	return SQLNullID{
		sql.NullString{
			String: string(id),
			Valid:  id != "",
		},
	}
}

type SQLNullString struct {
	sql.NullString
}

func (ns SQLNullString) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.String)
	}
	return json.Marshal(nil)
}

type SQLNullID struct {
	sql.NullString
}

func (ns SQLNullID) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.String)
	}
	return json.Marshal(nil)
}

type SQLNullInt64 struct {
	sql.NullInt64
}

func (ns SQLNullInt64) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.Int64)
	}
	return json.Marshal(nil)
}

type SQLNullFloat64 struct {
	sql.NullFloat64
}

func (ns SQLNullFloat64) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.Float64)
	}
	return json.Marshal(nil)
}

type SQLNullBool struct {
	sql.NullBool
}

func (ns SQLNullBool) MarshalJSON() ([]byte, error) {
	if ns.Valid {
		return json.Marshal(ns.Bool)
	}
	return json.Marshal(nil)
}

func NewSQLID() SQLID {
	var b [15]byte
	binary.BigEndian.PutUint32(b[:], uint32(time.Now().Unix()))

	b[4] = randID[0]
	b[5] = randID[1]
	b[6] = randID[2]
	b[7] = randID[3]
	b[8] = randID[4]

	b[9] = osPID[0]
	b[10] = osPID[1]

	binary.BigEndian.PutUint32(b[11:], atomic.AddUint32(&counter, 1))
	return SQLID(base32.StdEncoding.EncodeToString(b[:]))
}

func NewSQLNullID() SQLNullID {
	return SQLNullID{
		sql.NullString{
			String: "",
			Valid:  false,
		},
	}
}

func NewSQLNullString() SQLNullString {
	return SQLNullString{
		sql.NullString{
			String: "",
			Valid:  false,
		},
	}
}

func IsSQLID(id string) bool {
	return len(id) == 24
}

func SQLInsert(db SQLExecutor, table string, m interface{}) (sql.Result, error) {
	if m == nil {
		return nil, nil
	}

	var v reflect.Value

	if v = reflect.ValueOf(m); v.Type().Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if v.Type().Kind() != reflect.Struct {
		return nil, nil
	}

	var (
		t            = v.Type()
		l            = t.NumField()
		fields       = make([]string, 0, l)
		values       = make([]interface{}, 0, l)
		valuesholder = make([]string, 0, l)
		query        *sql.Stmt
		err          error
	)

	for i := 0; i < l; i++ {
		if field := t.Field(i); field.PkgPath == "" {
			fname := field.Tag.Get("db")
			if fname == "" {
				fname = strings.ToLower(field.Name)
			}
			iface := v.Field(i).Interface()
			if t, ok := iface.(time.Time); ok && t.IsZero() {
				continue
			}
			fields = append(fields, fname)
			values = append(values, iface)
			valuesholder = append(valuesholder, "?")
		}
	}
	stmt := fmt.Sprintf("INSERT INTO `%s`(`%s`) VALUES(%s)",
		table,
		strings.Join(fields, "`,`"),
		strings.Join(valuesholder, ","))

	if query, err = db.Prepare(stmt); err == nil {
		return query.Exec(values...)
	}

	return nil, err
}
<|endoftext|>


<commit_before>// +build windows

package windowsbase

import (
	"syscall"
	"unsafe"

	"golang.org/x/sys/windows"
)

var (
	shell32                  = windows.NewLazyDLL("Shell32.dll")
	ole32                    = windows.NewLazyDLL("Ole32.dll")
	procSHGetKnownFolderPath = modShell32.NewProc("SHGetKnownFolderPath")
	procCoTaskMemFree        = modOle32.NewProc("CoTaskMemFree")
)

func knownFolderDir(fid *FolderID) (string, error) {
	var path uintptr
	err := SHGetKnownFolderPath(fid, 0, 0, &path)
	if err != nil {
		return "", err
	}
	defer CoTaskMemFree(path)
	dir := syscall.UTF16ToString((*[1 << 16]uint16)(unsafe.Pointer(path))[:])
	return dir, nil
}

func shGetKnownFolderPath(fid *FolderID, dwFlags uint32, hToken syscall.Handle, pszPath *uintptr) (retval error) {
	r0, _, _ := procSHGetKnownFolderPath.Call(uintptr(unsafe.Pointer(rfid)), uintptr(dwFlags), uintptr(hToken), uintptr(unsafe.Pointer(pszPath)), 0, 0)
	if r0 != 0 {
		return syscall.Errno(r0)
	}
	return nil
}

func coTaskMemFree(pv uintptr) {
	procCoTaskMemFree.Call(uintptr(pv), 0, 0)
}
<commit_msg>windowsbase: Fix typos in the implementation<commit_after>// +build windows

package windowsbase

import (
	"syscall"
	"unsafe"

	"golang.org/x/sys/windows"
)

var (
	shell32                  = windows.NewLazyDLL("Shell32.dll")
	ole32                    = windows.NewLazyDLL("Ole32.dll")
	procSHGetKnownFolderPath = shell32.NewProc("SHGetKnownFolderPath")
	procCoTaskMemFree        = ole32.NewProc("CoTaskMemFree")
)

func knownFolderDir(fid *FolderID) (string, error) {
	var path uintptr
	err := shGetKnownFolderPath(fid, 0, 0, &path)
	if err != nil {
		return "", err
	}
	defer coTaskMemFree(path)
	dir := syscall.UTF16ToString((*[1 << 16]uint16)(unsafe.Pointer(path))[:])
	return dir, nil
}

func shGetKnownFolderPath(fid *FolderID, dwFlags uint32, hToken syscall.Handle, pszPath *uintptr) (retval error) {
	r0, _, _ := procSHGetKnownFolderPath.Call(uintptr(unsafe.Pointer(fid)), uintptr(dwFlags), uintptr(hToken), uintptr(unsafe.Pointer(pszPath)), 0, 0)
	if r0 != 0 {
		return syscall.Errno(r0)
	}
	return nil
}

func coTaskMemFree(pv uintptr) {
	procCoTaskMemFree.Call(uintptr(pv), 0, 0)
}
<|endoftext|>


<commit_before>package web

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"regexp"
	"testing"
	"time"
)

// These tests can probably be DRY'd up a bunch

func makeRouter() *router {
	return &router{
		routes:   make([]route, 0),
		notFound: parseHandler(http.NotFound),
	}
}

func chHandler(ch chan string, s string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- s
	})
}

var methods = []string{"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH",
	"POST", "PUT", "TRACE", "OTHER"}

func TestMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Connect("/", chHandler(ch, "CONNECT"))
	rt.Delete("/", chHandler(ch, "DELETE"))
	rt.Head("/", chHandler(ch, "HEAD"))
	rt.Get("/", chHandler(ch, "GET"))
	rt.Options("/", chHandler(ch, "OPTIONS"))
	rt.Patch("/", chHandler(ch, "PATCH"))
	rt.Post("/", chHandler(ch, "POST"))
	rt.Put("/", chHandler(ch, "PUT"))
	rt.Trace("/", chHandler(ch, "TRACE"))
	rt.Handle("/", chHandler(ch, "OTHER"))

	for _, method := range methods {
		r, _ := http.NewRequest(method, "/", nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case val := <-ch:
			if val != method {
				t.Errorf("Got %q, expected %q", val, method)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for method %q", method)
		}
	}
}

type testPattern struct{}

func (t testPattern) Prefix() string {
	return ""
}

func (t testPattern) Match(r *http.Request, c *C, dryrun bool) bool {
	return true
}

var _ Pattern = testPattern{}

func TestPatternTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	rt.Get("/hello/carl", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Get(regexp.MustCompile(`^/hello/(?P<name>.+)$`), http.NotFound)
	rt.Get(testPattern{}, http.NotFound)
}

type testHandler chan string

func (t testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t <- "http"
}
func (t testHandler) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {
	t <- "httpc"
}

var testHandlerTable = map[string]string{
	"/a": "http fn",
	"/b": "http handler",
	"/c": "web fn",
	"/d": "web handler",
	"/e": "httpc",
}

func TestHandlerTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Get("/a", func(w http.ResponseWriter, r *http.Request) {
		ch <- "http fn"
	})
	rt.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- "http handler"
	}))
	rt.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web fn"
	})
	rt.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web handler"
	}))
	rt.Get("/e", testHandler(ch))

	for route, response := range testHandlerTable {
		r, _ := http.NewRequest("GET", route, nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case resp := <-ch:
			if resp != response {
				t.Errorf("Got %q, expected %q", resp, response)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for path %q", route)
		}

	}
}

func TestNotFound(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	r, _ := http.NewRequest("post", "/", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != 404 {
		t.Errorf("Expected 404, got %d", w.Code)
	}

	rt.NotFound(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "I'm a teapot!", http.StatusTeapot)
	})

	r, _ = http.NewRequest("POST", "/", nil)
	w = httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != http.StatusTeapot {
		t.Errorf("Expected a teapot, got %d", w.Code)
	}
}

func TestPrefix(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Handle("/hello*", func(w http.ResponseWriter, r *http.Request) {
		ch <- r.URL.Path
	})

	r, _ := http.NewRequest("GET", "/hello/world", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	select {
	case val := <-ch:
		if val != "/hello/world" {
			t.Errorf("Got %q, expected /hello/world", val)
		}
	case <-time.After(5 * time.Millisecond):
		t.Errorf("Timeout waiting for hello")
	}
}

var validMethodsTable = map[string][]string{
	"/hello/carl":       {"DELETE", "GET", "HEAD", "PATCH", "POST", "PUT"},
	"/hello/bob":        {"DELETE", "GET", "HEAD", "PATCH", "PUT"},
	"/hola/carl":        {"DELETE", "GET", "HEAD", "PUT"},
	"/hola/bob":         {"DELETE"},
	"/does/not/compute": {},
}

func TestValidMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan []string, 1)

	rt.NotFound(func(c C, w http.ResponseWriter, r *http.Request) {
		if c.Env == nil {
			ch <- []string{}
			return
		}
		methods, ok := c.Env[ValidMethodsKey]
		if !ok {
			ch <- []string{}
			return
		}
		ch <- methods.([]string)
	})

	rt.Get("/hello/carl", http.NotFound)
	rt.Post("/hello/carl", http.NotFound)
	rt.Head("/hello/bob", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Put("/hello/:name", http.NotFound)
	rt.Patch("/hello/:name", http.NotFound)
	rt.Get("/:greet/carl", http.NotFound)
	rt.Put("/:greet/carl", http.NotFound)
	rt.Delete("/:greet/:anyone", http.NotFound)

	for path, eMethods := range validMethodsTable {
		r, _ := http.NewRequest("BOGUS", path, nil)
		rt.route(C{}, httptest.NewRecorder(), r)
		aMethods := <-ch
		if !reflect.DeepEqual(eMethods, aMethods) {
			t.Errorf("For %q, expected %v, got %v", path, eMethods,
				aMethods)
		}
	}
}
<commit_msg>Add route selection test<commit_after>package web

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"regexp"
	"testing"
	"time"
)

// These tests can probably be DRY'd up a bunch

func makeRouter() *router {
	return &router{
		routes:   make([]route, 0),
		notFound: parseHandler(http.NotFound),
	}
}

func chHandler(ch chan string, s string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- s
	})
}

var methods = []string{"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH",
	"POST", "PUT", "TRACE", "OTHER"}

func TestMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Connect("/", chHandler(ch, "CONNECT"))
	rt.Delete("/", chHandler(ch, "DELETE"))
	rt.Head("/", chHandler(ch, "HEAD"))
	rt.Get("/", chHandler(ch, "GET"))
	rt.Options("/", chHandler(ch, "OPTIONS"))
	rt.Patch("/", chHandler(ch, "PATCH"))
	rt.Post("/", chHandler(ch, "POST"))
	rt.Put("/", chHandler(ch, "PUT"))
	rt.Trace("/", chHandler(ch, "TRACE"))
	rt.Handle("/", chHandler(ch, "OTHER"))

	for _, method := range methods {
		r, _ := http.NewRequest(method, "/", nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case val := <-ch:
			if val != method {
				t.Errorf("Got %q, expected %q", val, method)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for method %q", method)
		}
	}
}

type testPattern struct{}

func (t testPattern) Prefix() string {
	return ""
}

func (t testPattern) Match(r *http.Request, c *C, dryrun bool) bool {
	return true
}

var _ Pattern = testPattern{}

func TestPatternTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	rt.Get("/hello/carl", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Get(regexp.MustCompile(`^/hello/(?P<name>.+)$`), http.NotFound)
	rt.Get(testPattern{}, http.NotFound)
}

type testHandler chan string

func (t testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t <- "http"
}
func (t testHandler) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {
	t <- "httpc"
}

var testHandlerTable = map[string]string{
	"/a": "http fn",
	"/b": "http handler",
	"/c": "web fn",
	"/d": "web handler",
	"/e": "httpc",
}

func TestHandlerTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Get("/a", func(w http.ResponseWriter, r *http.Request) {
		ch <- "http fn"
	})
	rt.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- "http handler"
	}))
	rt.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web fn"
	})
	rt.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web handler"
	}))
	rt.Get("/e", testHandler(ch))

	for route, response := range testHandlerTable {
		r, _ := http.NewRequest("GET", route, nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case resp := <-ch:
			if resp != response {
				t.Errorf("Got %q, expected %q", resp, response)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for path %q", route)
		}

	}
}

// The idea behind this test is to comprehensively test if routes are being
// applied in the right order. We define a special pattern type that always
// matches so long as it's greater than or equal to the global test index. By
// incrementing this index, we can invalidate all routes up to some point, and
// therefore test the routing guarantee that Goji provides: for any path P, if
// both A and B match P, and if A was inserted before B, then Goji will route to
// A before it routes to B.
var rsRoutes = []string{
	"/",
	"/a",
	"/a",
	"/b",
	"/ab",
	"/",
	"/ba",
	"/b",
	"/a",
}

var rsTests = []struct {
	key     string
	results []int
}{
	{"/", []int{0, 5, 5, 5, 5, 5, -1, -1, -1, -1}},
	{"/a", []int{0, 1, 2, 5, 5, 5, 8, 8, 8, -1}},
	{"/b", []int{0, 3, 3, 3, 5, 5, 7, 7, -1, -1}},
	{"/ab", []int{0, 1, 2, 4, 4, 5, 8, 8, 8, -1}},
	{"/ba", []int{0, 3, 3, 3, 5, 5, 6, 7, -1, -1}},
	{"/c", []int{0, 5, 5, 5, 5, 5, -1, -1, -1, -1}},
	{"nope", []int{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}},
}

type rsPattern struct {
	i       int
	counter *int
	prefix  string
	ichan   chan int
}

func (rs rsPattern) Prefix() string {
	return rs.prefix
}
func (rs rsPattern) Match(_ *http.Request, _ *C, _ bool) bool {
	return rs.i >= *rs.counter
}

func (rs rsPattern) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {
	rs.ichan <- rs.i
}

var _ Pattern = rsPattern{}
var _ http.Handler = rsPattern{}

func TestRouteSelection(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	counter := 0
	ichan := make(chan int, 1)
	rt.NotFound(func(w http.ResponseWriter, r *http.Request) {
		ichan <- -1
	})

	for i, s := range rsRoutes {
		pat := rsPattern{
			i:       i,
			counter: &counter,
			prefix:  s,
			ichan:   ichan,
		}
		rt.Get(pat, pat)
	}

	for _, test := range rsTests {
		var n int
		for counter, n = range test.results {
			r, _ := http.NewRequest("GET", test.key, nil)
			w := httptest.NewRecorder()
			rt.route(C{}, w, r)
			actual := <-ichan
			if n != actual {
				t.Errorf("Expected %q @ %d to be %d, got %d",
					test.key, counter, n, actual)
			}
		}
	}
}

func TestNotFound(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	r, _ := http.NewRequest("post", "/", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != 404 {
		t.Errorf("Expected 404, got %d", w.Code)
	}

	rt.NotFound(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "I'm a teapot!", http.StatusTeapot)
	})

	r, _ = http.NewRequest("POST", "/", nil)
	w = httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != http.StatusTeapot {
		t.Errorf("Expected a teapot, got %d", w.Code)
	}
}

func TestPrefix(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Handle("/hello*", func(w http.ResponseWriter, r *http.Request) {
		ch <- r.URL.Path
	})

	r, _ := http.NewRequest("GET", "/hello/world", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	select {
	case val := <-ch:
		if val != "/hello/world" {
			t.Errorf("Got %q, expected /hello/world", val)
		}
	case <-time.After(5 * time.Millisecond):
		t.Errorf("Timeout waiting for hello")
	}
}

var validMethodsTable = map[string][]string{
	"/hello/carl":       {"DELETE", "GET", "HEAD", "PATCH", "POST", "PUT"},
	"/hello/bob":        {"DELETE", "GET", "HEAD", "PATCH", "PUT"},
	"/hola/carl":        {"DELETE", "GET", "HEAD", "PUT"},
	"/hola/bob":         {"DELETE"},
	"/does/not/compute": {},
}

func TestValidMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan []string, 1)

	rt.NotFound(func(c C, w http.ResponseWriter, r *http.Request) {
		if c.Env == nil {
			ch <- []string{}
			return
		}
		methods, ok := c.Env[ValidMethodsKey]
		if !ok {
			ch <- []string{}
			return
		}
		ch <- methods.([]string)
	})

	rt.Get("/hello/carl", http.NotFound)
	rt.Post("/hello/carl", http.NotFound)
	rt.Head("/hello/bob", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Put("/hello/:name", http.NotFound)
	rt.Patch("/hello/:name", http.NotFound)
	rt.Get("/:greet/carl", http.NotFound)
	rt.Put("/:greet/carl", http.NotFound)
	rt.Delete("/:greet/:anyone", http.NotFound)

	for path, eMethods := range validMethodsTable {
		r, _ := http.NewRequest("BOGUS", path, nil)
		rt.route(C{}, httptest.NewRecorder(), r)
		aMethods := <-ch
		if !reflect.DeepEqual(eMethods, aMethods) {
			t.Errorf("For %q, expected %v, got %v", path, eMethods,
				aMethods)
		}
	}
}
<|endoftext|>


<commit_before>package loaders

import (
	"testing"
)

func TestTextfileLoaderAvailable(t *testing.T) {
	var testfile = make(map[string]string)
	testfile["path"] = "./example.urls.txt" // target known file
	if !textfileLoaderAvailable(testfile) {
		t.Fatal("textfileLoaderAvailable failed to find known file")
	}
}
<commit_msg>Added regex tests<commit_after>package loaders

import (
	"testing"
)

func TestTextfileLoaderAvailable(t *testing.T) {
	var testfile = make(map[string]string)
	testfile["path"] = "./example.urls.txt" // target known file
	if !textfileLoaderAvailable(testfile) {
		t.Fatal("textfileLoaderAvailable failed to find known file")
	}
}

func TestParseProjectsOutput(t *testing.T) {
	if parseProjectsOutput("test-name   other-thing  a") != "test-name" {
		t.Error("Was expecting a different output")
	}
}

func TestParseModulesOutput(t *testing.T) {
	if parseModulesOutput("test-name   other-thing  a") != "other-thing" {
		t.Error("Was expecting a different output")
	}
}
<|endoftext|>


<commit_before>package model

import (
	"errors"
	"time"
)

type Job struct {
	ID              uint64 `gorm:"column:id; primary_key:yes"`
	StartTime       time.Time `sql:"DEFAULT:null"`
	EndTime         time.Time `sql:"DEFAULT:null"`
	EmployerName    string
	LeaveAllowances []LeaveAllowance
	LeaveRequests   []LeaveRequest

	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt time.Time
}

type LeaveAllowance struct {
	ID        uint64    `gorm:"column:id; primary_key:yes"`
	StartDate time.Time `sql:"DEFAULT:null"`
	Days      float64
	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt time.Time
}

type LeaveRequest struct {
	ID          uint64 `gorm:"column:id; primary_key:yes"`
	AddedBy     User
	Days        float64
	Description string
	StartDate   time.Time `sql:"DEFAULT:null"`
}

type User struct {
	ID       uint64 `gorm:"column:id; primary_key:yes"`
	Name     string `sql:"type:text;"`
	GitHubID uint64
	Email    string `sql:"type:text;"`
	TZOffset int16  // time zone as seconds east of UTC
	Jobs     []Job

	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt time.Time
}

func (u *User) UpdateOrCreate() error {
	if u.GitHubID == 0 {
		return errors.New("GitHub user ID was set to zero; cannot match")
	}

	res := db.Where(User{GitHubID: u.GitHubID}).FirstOrInit(u)

	if res.Error != nil {
		return res.Error
	}

	res = db.Save(u)
	return res.Error
}

func FindUser(id uint64) (user User, err error) {
	res := db.First(&user, id)
	return user, res.Error
}
<commit_msg>Fix struct field tag indentation<commit_after>package model

import (
	"errors"
	"time"
)

type Job struct {
	ID              uint64    `gorm:"column:id; primary_key:yes"`
	StartTime       time.Time `sql:"DEFAULT:null"`
	EndTime         time.Time `sql:"DEFAULT:null"`
	EmployerName    string
	LeaveAllowances []LeaveAllowance
	LeaveRequests   []LeaveRequest

	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt time.Time
}

type LeaveAllowance struct {
	ID        uint64    `gorm:"column:id; primary_key:yes"`
	StartDate time.Time `sql:"DEFAULT:null"`
	Days      float64
	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt time.Time
}

type LeaveRequest struct {
	ID          uint64 `gorm:"column:id; primary_key:yes"`
	AddedBy     User
	Days        float64
	Description string
	StartDate   time.Time `sql:"DEFAULT:null"`
}

type User struct {
	ID       uint64 `gorm:"column:id; primary_key:yes"`
	Name     string `sql:"type:text;"`
	GitHubID uint64
	Email    string `sql:"type:text;"`
	TZOffset int16  // time zone as seconds east of UTC
	Jobs     []Job

	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt time.Time
}

func (u *User) UpdateOrCreate() error {
	if u.GitHubID == 0 {
		return errors.New("GitHub user ID was set to zero; cannot match")
	}

	res := db.Where(User{GitHubID: u.GitHubID}).FirstOrInit(u)

	if res.Error != nil {
		return res.Error
	}

	res = db.Save(u)
	return res.Error
}

func FindUser(id uint64) (user User, err error) {
	res := db.First(&user, id)
	return user, res.Error
}
<|endoftext|>


<commit_before>package formatter

import (
	"github.com/BluePecker/JwtAuth/dialog/client/formatter/context"
	"github.com/BluePecker/JwtAuth/dialog/server/parameter/jwt/response"
	"bytes"
	"strconv"
)

const (
	AddrHeader   = "ADDR"
	TLLHeader    = "TLL"
	DeviceHeader = "DEVICE"
	SingedHeader = "SINGED"

	QuietFormat    = "{{.Singed}}"
	JwtTableFormat = "table {{.Addr}}\t{{.TTL}}\t{{.Device}}\t{{.Singed}}"
)

type (
	JsonWebToken struct {
		context.BaseSubjectContext
		truncate bool
		jwt      response.JsonWebToken
	}

	JsonWebTokenContext struct {
		context.Context
		JsonWebTokens []response.JsonWebToken
	}
)

func (ctx JsonWebTokenContext) Write() {
	switch ctx.Template {
	case context.RawKey:
		if ctx.Quiet {
			ctx.Template = `Singed: {{.Singed}}`
		} else {
			ctx.Template = `Addr: {{.Addr}}\nTTL: {{.TTL}}\nDevice: {{.Device}}\nSinged: {{.Singed}}\n`
		}
	case context.TableKey:
		if ctx.Quiet {
			ctx.Template = QuietFormat
		} else {
			ctx.Template = JwtTableFormat
		}
	}

	ctx.Buffer = bytes.NewBufferString("")
	ctx.PreFormat()

	tpl, err := ctx.Parser()
	if err != nil {
		return
	}

	for _, jwt := range ctx.JsonWebTokens {
		jwtCtx := &JsonWebToken{
			truncate: ctx.Truncate,
			jwt:      jwt,
		}
		err = ctx.ContextFormat(tpl, jwtCtx)
		if err != nil {
			return
		}
	}

	ctx.FormFormat(tpl, &JsonWebToken{})
}

func (j *JsonWebToken) Addr() string {
	j.AddHeader(AddrHeader)
	return j.jwt.Addr
}

func (j *JsonWebToken) TLL() string {
	j.AddHeader(TLLHeader)
	return strconv.FormatFloat(j.jwt.TTL, 'E', -1, 64)
}

func (j *JsonWebToken) Device() string {
	j.AddHeader(DeviceHeader)
	return j.jwt.Device
}

func (j *JsonWebToken) Singed() string {
	j.AddHeader(SingedHeader)
	return j.jwt.Singed
}
<commit_msg>debug<commit_after>package formatter

import (
	"github.com/BluePecker/JwtAuth/dialog/client/formatter/context"
	"github.com/BluePecker/JwtAuth/dialog/server/parameter/jwt/response"
	"bytes"
	"strconv"
)

const (
	AddrHeader   = "ADDR"
	TLLHeader    = "TLL"
	DeviceHeader = "DEVICE"
	SingedHeader = "SINGED"

	QuietFormat    = "{{.Singed}}"
	JwtTableFormat = "table {{.Addr}}\t{{.TTL}}\t{{.Device}}\t{{.Singed}}"
)

type (
	JsonWebToken struct {
		context.BaseSubjectContext
		truncate bool
		jwt      response.JsonWebToken
	}

	JsonWebTokenContext struct {
		context.Context
		JsonWebTokens []response.JsonWebToken
	}
)

func (ctx JsonWebTokenContext) Write() {
	switch ctx.Template {
	case context.RawKey:
		if ctx.Quiet {
			ctx.Template = `Singed: {{.Singed}}`
		} else {
			ctx.Template = `Addr: {{.Addr}}\nTTL: {{.TTL}}\nDevice: {{.Device}}\nSinged: {{.Singed}}\n`
		}
	case context.TableKey:
		if ctx.Quiet {
			ctx.Template = QuietFormat
		} else {
			ctx.Template = JwtTableFormat
		}
	}

	ctx.Buffer = bytes.NewBufferString("")
	ctx.PreFormat()

	tpl, err := ctx.Parser()
	if err != nil {
		return
	}

	for _, jwt := range ctx.JsonWebTokens {
		jwtCtx := &JsonWebToken{
			truncate: ctx.Truncate,
			jwt:      jwt,
		}
		err = ctx.ContextFormat(tpl, jwtCtx)
		if err != nil {
			return
		}
	}

	ctx.FormFormat(tpl, &JsonWebToken{})
}

func (j *JsonWebToken) Addr() string {
	j.AddHeader(AddrHeader)
	return j.jwt.Addr
}

func (j *JsonWebToken) TLL() string {
	j.AddHeader(TLLHeader)
	return strconv.FormatFloat(j.jwt.TTL, 'f', -1, 64)
}

func (j *JsonWebToken) Device() string {
	j.AddHeader(DeviceHeader)
	return j.jwt.Device
}

func (j *JsonWebToken) Singed() string {
	j.AddHeader(SingedHeader)
	return j.jwt.Singed
}
<|endoftext|>