<commit_before>package dialects

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/wunderlist/hamustro/src/payload"
	"reflect"
	"testing"
)

// Returns an Event for testing purposes
func GetTestEvent(userId uint32) *Event {
	return &Event{
		DeviceID:        "a73b1c37-2c24-4786-af7a-16de88fbe23a",
		ClientID:        "bce44f67b2661fd445d469b525b04f68",
		Session:         "244f056dee6d475ec673ea0d20b69bab",
		Nr:              1,
		Env:             "PRODUCTION",
		SystemVersion:   "10.10",
		ProductVersion:  "1.1.2",
		At:              "2016-02-05T15:05:04",
		Timezone:        "+02:00",
		Event:           "Client.CreateUser",
		DeviceMake:      "Iphone",
		DeviceModel:     "Iphone 6",
		System:          "OSX",
		SystemLanguage:  "DE",
		Browser:         "Mozilla",
		BrowserVersion:  "10.01.11",
		ProductGitHash:  "5416a5889392d509e3bafcf40f6388e83aab23e6",
		ProductLanguage: "HU",
		UserID:          fmt.Sprintf("%v", userId),
		TenantID:        "sdfghjkloiuytremiwoz",
		IP:              "214.160.227.22",
		Country:         "UK",
		Parameters:      "{parameter: test_parameter}"}
}

// Converts and Event into a list of string
func TestEventStringConversion(t *testing.T) {
	t.Log("Converting Event into a list of strings")
	e := GetTestEvent(97421193)
	exp := []string{
		"a73b1c37-2c24-4786-af7a-16de88fbe23a",
		"bce44f67b2661fd445d469b525b04f68",
		"244f056dee6d475ec673ea0d20b69bab",
		"1",
		"PRODUCTION",
		"10.10",
		"1.1.2",
		"2016-02-05T15:05:04",
		"+02:00",
		"Client.CreateUser",
		"Iphone",
		"Iphone 6",
		"OSX",
		"DE",
		"Mozilla",
		"10.01.11",
		"5416a5889392d509e3bafcf40f6388e83aab23e6",
		"HU",
		"97421193",
		"sdfghjkloiuytremiwoz",
		"214.160.227.22",
		"UK",
		"{parameter: test_parameter}"}
	if !reflect.DeepEqual(e.String(), exp) {
		t.Error("Expected event's string is not matched")
	}
}

// Tests the IP truncating functionality
func TestFunctionTruncateIPv4LastOctet(t *testing.T) {
	cases := []struct {
		IP         string
		ExpectedIP string
	}{
		{"214.160.227.22", "214.160.227.0"},
		{"214.160.227.22:80", "214.160.227.0"},
		{"214.160.227.22/24", "214.160.227.0"},
		{"", ""}}
	t.Log("Removing last octet of IP address")
	for _, c := range cases {
		e := &Event{IP: c.IP}
		e.TruncateIPv4LastOctet()
		if c.ExpectedIP != e.IP {
			t.Errorf("Expected truncated IP was %s but it was %s instead", c.ExpectedIP, e.IP)
		}
	}
}

// Tests the Events creation from a Collection.
func TestNewEventCreation(t *testing.T) {
	t.Log("Creating a new Event object from Payload and Collection")
	env := payload.Environment_PRODUCTION
	p := payload.Payload{
		At:         proto.Uint64(1454681104),
		Event:      proto.String("Client.CreateUser"),
		Nr:         proto.Uint32(1),
		UserId:     proto.String("97421193"),
		Ip:         proto.String("214.160.227.22"),
		Parameters: proto.String("")}
	c := payload.Collection{
		DeviceId:       proto.String("a73b1c37-2c24-4786-af7a-16de88fbe23a"),
		ClientId:       proto.String("bce44f67b2661fd445d469b525b04f68"),
		Session:        proto.String("244f056dee6d475ec673ea0d20b69bab"),
		SystemVersion:  proto.String("10.10"),
		ProductVersion: proto.String("1.1.2"),
		System:         proto.String("OSX"),
		ProductGitHash: proto.String("5416a5889392d509e3bafcf40f6388e83aab23e6"),
		Env:            &env,
		Payloads:       []*payload.Payload{&p}}
	e := NewEvent(&c, &p)

	if exp := "a73b1c37-2c24-4786-af7a-16de88fbe23a"; e.DeviceID != exp {
		t.Errorf("Expected DeviceID was %s but it was %s instead", exp, e.DeviceID)
	}
	if exp := "bce44f67b2661fd445d469b525b04f68"; e.ClientID != exp {
		t.Errorf("Expected ClientID was %s but it was %s instead", exp, e.ClientID)
	}
	if exp := "244f056dee6d475ec673ea0d20b69bab"; e.Session != exp {
		t.Errorf("Expected Session was %s but it was %s instead", exp, e.Session)
	}
	if exp := uint32(1); e.Nr != exp {
		t.Errorf("Expected Nr was %s but it was %d instead", exp, e.Nr)
	}
	if exp := "10.10"; e.SystemVersion != exp {
		t.Errorf("Expected SystemVersion was %s but it was %s instead", exp, e.SystemVersion)
	}
	if exp := "1.1.2"; e.ProductVersion != exp {
		t.Errorf("Expected ProductVersion was %s but it was %s instead", exp, e.ProductVersion)
	}
	if exp := ConvertIsoformat(1454681104); e.At != exp {
		t.Errorf("Expected At was %s but it was %s instead", exp, e.At)
	}
	if exp := "Client.CreateUser"; e.Event != exp {
		t.Errorf("Expected Event was %s but it was %s instead", exp, e.Event)
	}
	if exp := "OSX"; e.System != exp {
		t.Errorf("Expected System was %s but it was %s instead", exp, e.System)
	}
	if exp := "5416a5889392d509e3bafcf40f6388e83aab23e6"; e.ProductGitHash != exp {
		t.Errorf("Expected ProductGitHash was %s but it was %s instead", exp, e.ProductGitHash)
	}
	if exp := "97421193"; e.UserID != exp {
		t.Errorf("Expected UserID was %s but it was %d instead", exp, e.UserID)
	}
	if exp := "214.160.227.22"; e.IP != exp {
		t.Errorf("Expected IP was %s but it was %s instead", exp, e.IP)
	}
	if exp := ""; e.Parameters != exp {
		t.Errorf("Expected Parameters was %s but it was %s instead", exp, e.Parameters)
	}
	if exp := "PRODUCTION"; e.Env != exp {
		t.Errorf("Expected Env was %s but it was %s instead", exp, e.Env)
	}
}
<commit_msg>event paramter tests<commit_after>package dialects

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/wunderlist/hamustro/src/payload"
	"reflect"
	"testing"
)

// Returns an Event for testing purposes
func GetTestEvent(userId uint32) *Event {
	return &Event{
		DeviceID:        "a73b1c37-2c24-4786-af7a-16de88fbe23a",
		ClientID:        "bce44f67b2661fd445d469b525b04f68",
		Session:         "244f056dee6d475ec673ea0d20b69bab",
		Nr:              1,
		Env:             "PRODUCTION",
		SystemVersion:   "10.10",
		ProductVersion:  "1.1.2",
		At:              "2016-02-05T15:05:04",
		Timezone:        "+02:00",
		Event:           "Client.CreateUser",
		DeviceMake:      "Iphone",
		DeviceModel:     "Iphone 6",
		System:          "OSX",
		SystemLanguage:  "DE",
		Browser:         "Mozilla",
		BrowserVersion:  "10.01.11",
		ProductGitHash:  "5416a5889392d509e3bafcf40f6388e83aab23e6",
		ProductLanguage: "HU",
		UserID:          fmt.Sprintf("%v", userId),
		TenantID:        "sdfghjkloiuytremiwoz",
		IP:              "214.160.227.22",
		Country:         "UK",
		Parameters:      "{parameter: test_parameter}"}
}

// Converts and Event into a list of string
func TestEventStringConversion(t *testing.T) {
	t.Log("Converting Event into a list of strings")
	e := GetTestEvent(97421193)
	exp := []string{
		"a73b1c37-2c24-4786-af7a-16de88fbe23a",
		"bce44f67b2661fd445d469b525b04f68",
		"244f056dee6d475ec673ea0d20b69bab",
		"1",
		"PRODUCTION",
		"10.10",
		"1.1.2",
		"2016-02-05T15:05:04",
		"+02:00",
		"Client.CreateUser",
		"Iphone",
		"Iphone 6",
		"OSX",
		"DE",
		"Mozilla",
		"10.01.11",
		"5416a5889392d509e3bafcf40f6388e83aab23e6",
		"HU",
		"97421193",
		"sdfghjkloiuytremiwoz",
		"214.160.227.22",
		"UK",
		"{parameter: test_parameter}"}
	if !reflect.DeepEqual(e.String(), exp) {
		t.Error("Expected event's string is not matched")
	}
}

// Tests the IP truncating functionality
func TestFunctionTruncateIPv4LastOctet(t *testing.T) {
	cases := []struct {
		IP         string
		ExpectedIP string
	}{
		{"214.160.227.22", "214.160.227.0"},
		{"214.160.227.22:80", "214.160.227.0"},
		{"214.160.227.22/24", "214.160.227.0"},
		{"", ""}}
	t.Log("Removing last octet of IP address")
	for _, c := range cases {
		e := &Event{IP: c.IP}
		e.TruncateIPv4LastOctet()
		if c.ExpectedIP != e.IP {
			t.Errorf("Expected truncated IP was %s but it was %s instead", c.ExpectedIP, e.IP)
		}
	}
}

// Tests the Events creation from a Collection.
func TestNewEventCreation(t *testing.T) {
	t.Log("Creating a new Event object from Payload and Collection")
	env := payload.Environment_PRODUCTION
	p := payload.Payload{
		At:         proto.Uint64(1454681104),
		Timezone:   proto.String("+02:00"),
		Event:      proto.String("Client.CreateUser"),
		Nr:         proto.Uint32(1),
		UserId:     proto.String("97421193"),
		TenantId:   proto.String("sdfghjkloiuytremiwoz"),
		Ip:         proto.String("214.160.227.22"),
		Country:    proto.String("UK"),
		Parameters: proto.String("{parameter: test_parameter}")}
	c := payload.Collection{
		DeviceId:        proto.String("a73b1c37-2c24-4786-af7a-16de88fbe23a"),
		ClientId:        proto.String("bce44f67b2661fd445d469b525b04f68"),
		Session:         proto.String("244f056dee6d475ec673ea0d20b69bab"),
		Env:             &env,
		SystemVersion:   proto.String("10.10"),
		ProductVersion:  proto.String("1.1.2"),
		DeviceMake:      proto.String("Iphone"),
		DeviceModel:     proto.String("Iphone 6"),
		System:          proto.String("OSX"),
		SystemLanguage:  proto.String("DE"),
		Browser:         proto.String("Mozilla"),
		BrowserVersion:  proto.String("10.01.11"),
		ProductGitHash:  proto.String("5416a5889392d509e3bafcf40f6388e83aab23e6"),
		ProductLanguage: proto.String("HU"),
		Payloads:        []*payload.Payload{&p}}
	e := NewEvent(&c, &p)

	if exp := ConvertIsoformat(1454681104); e.At != exp {
		t.Errorf("Expected At was %s but it was %s instead", exp, e.At)
	}
	if exp := "+02:00"; e.Timezone != exp {
		t.Errorf("Expected Timezone was %s but it was %s instead", exp, e.Timezone)
	}
	if exp := "Client.CreateUser"; e.Event != exp {
		t.Errorf("Expected Event was %s but it was %s instead", exp, e.Event)
	}
	if exp := uint32(1); e.Nr != exp {
		t.Errorf("Expected Nr was %s but it was %d instead", exp, e.Nr)
	}
	if exp := "97421193"; e.UserID != exp {
		t.Errorf("Expected UserID was %s but it was %d instead", exp, e.UserID)
	}
	if exp := "sdfghjkloiuytremiwoz"; e.TenantID != exp {
		t.Errorf("Expected TenantID was %s but it was %d instead", exp, e.TenantID)
	}
	if exp := "214.160.227.22"; e.IP != exp {
		t.Errorf("Expected IP was %s but it was %s instead", exp, e.IP)
	}
	if exp := "UK"; e.Country != exp {
		t.Errorf("Expected Country was %s but it was %s instead", exp, e.Country)
	}
	if exp := "{parameter: test_parameter}"; e.Parameters != exp {
		t.Errorf("Expected Parameters was %s but it was %s instead", exp, e.Parameters)
	}
	if exp := "a73b1c37-2c24-4786-af7a-16de88fbe23a"; e.DeviceID != exp {
		t.Errorf("Expected DeviceID was %s but it was %s instead", exp, e.DeviceID)
	}
	if exp := "bce44f67b2661fd445d469b525b04f68"; e.ClientID != exp {
		t.Errorf("Expected ClientID was %s but it was %s instead", exp, e.ClientID)
	}
	if exp := "244f056dee6d475ec673ea0d20b69bab"; e.Session != exp {
		t.Errorf("Expected Session was %s but it was %s instead", exp, e.Session)
	}
	if exp := "PRODUCTION"; e.Env != exp {
		t.Errorf("Expected Env was %s but it was %s instead", exp, e.Env)
	}
	if exp := "10.10"; e.SystemVersion != exp {
		t.Errorf("Expected SystemVersion was %s but it was %s instead", exp, e.SystemVersion)
	}
	if exp := "Iphone"; e.DeviceMake != exp {
		t.Errorf("Expected DeviceMake was %s but it was %s instead", exp, e.DeviceMake)
	}
	if exp := "Iphone 6"; e.DeviceModel != exp {
		t.Errorf("Expected DeviceModel was %s but it was %s instead", exp, e.DeviceModel)
	}
	if exp := "OSX"; e.System != exp {
		t.Errorf("Expected System was %s but it was %s instead", exp, e.System)
	}
	if exp := "DE"; e.SystemLanguage != exp {
		t.Errorf("Expected SystemLanguage was %s but it was %s instead", exp, e.SystemLanguage)
	}
	if exp := "Mozilla"; e.Browser != exp {
		t.Errorf("Expected Browser was %s but it was %s instead", exp, e.Browser)
	}
	if exp := "10.01.11"; e.BrowserVersion != exp {
		t.Errorf("Expected BrowserVersion was %s but it was %s instead", exp, e.BrowserVersion)
	}
	if exp := "5416a5889392d509e3bafcf40f6388e83aab23e6"; e.ProductGitHash != exp {
		t.Errorf("Expected ProductGitHash was %s but it was %s instead", exp, e.ProductGitHash)
	}
	if exp := "HU"; e.ProductLanguage != exp {
		t.Errorf("Expected ProductLanguage was %s but it was %s instead", exp, e.ProductLanguage)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/alecthomas/kingpin"
	"github.com/parnurzeal/gorequest"
	"github.com/toqueteos/webbrowser"
)

const htmlURL string = "http://kaifa.at/services/"
const jsonURL string = "http://kaifa.at/api/services/"

var (
	keyword = kingpin.Arg("keyword", "Keyword of service").Required().String()
	format  = kingpin.Flag("format", "Format: html,json").Default("html").String()
)

func main() {
	kingpin.Version("0.1.0")
	kingpin.Parse()

	switch *format {
	case "html":
		openBrowser()
	case "json":
		fetchJSON()
	default:
		openBrowser()
	}
}

func openBrowser() {
	url := fmt.Sprint(htmlURL, *keyword)
	webbrowser.Open(url)
}

func fetchJSON() {
	url := fmt.Sprint(jsonURL, *keyword)

	request := gorequest.New()
	resp, body, errs := request.Get(url).
		Set("Accept", "application/json").
		End()

	if len(errs) > 0 {
		fmt.Fprintf(os.Stderr, "Unexpected errors: %s", errs)
		os.Exit(1)
	}

	if resp.StatusCode != 200 {
		fmt.Fprintf(os.Stderr, "Expected StatusCode=200, actual StatusCode=%v", resp.StatusCode)
		os.Exit(2)
	}

	fmt.Printf(body)
}
<commit_msg>Bump to v0.2.0.<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/alecthomas/kingpin"
	"github.com/parnurzeal/gorequest"
	"github.com/toqueteos/webbrowser"
)

const htmlURL string = "http://kaifa.at/services/"
const jsonURL string = "http://kaifa.at/api/services/"

var (
	keyword = kingpin.Arg("keyword", "Keyword of service").Required().String()
	format  = kingpin.Flag("format", "Format: html,json").Default("html").String()
)

func main() {
	kingpin.Version("0.2.0")
	kingpin.Parse()

	switch *format {
	case "html":
		openBrowser()
	case "json":
		fetchJSON()
	default:
		openBrowser()
	}
}

func openBrowser() {
	url := fmt.Sprint(htmlURL, *keyword)
	webbrowser.Open(url)
}

func fetchJSON() {
	url := fmt.Sprint(jsonURL, *keyword)

	request := gorequest.New()
	resp, body, errs := request.Get(url).
		Set("Accept", "application/json").
		End()

	if len(errs) > 0 {
		fmt.Fprintf(os.Stderr, "Unexpected errors: %s", errs)
		os.Exit(1)
	}

	if resp.StatusCode != 200 {
		fmt.Fprintf(os.Stderr, "Expected StatusCode=200, actual StatusCode=%v", resp.StatusCode)
		os.Exit(2)
	}

	fmt.Printf(body)
}
<|endoftext|>


<commit_before>// Package loggedcache provides a cache wrapper that logs and records metrics
// about operations performed on an underlying cache.
package loggedcache

import (
	"log"
	"time"
)

// Async is a cache wrapper that asynchronously logs and records metrics about
// synchronous operations performed on an underlying cache.
type Async struct {
	// Underlying is the underlying cache to forward cache operations to.
	Underlying interface {
		Get(key string) (responseBytes []byte, ok bool)
		Set(key string, responseBytes []byte)
		Delete(key string)
	}

	// Count is called with "get"/"set"/"delete" for each operation performed on
	// the cache, even if it is unsuccessful.
	Count func(operation string)

	// Hit is called on each Get operation that returns data (i.e., ok is
	// true).
	Hit func()

	// Time is called with "get"/"set"/"delete" and the total duration for each
	// operation performed on the cache.
	Time func(operation string, t time.Duration)

	// Log receives log messages for operations performed on the underlying cache.
	Log *log.Logger
}

func (c *Async) Get(key string) (resp []byte, ok bool) {
	if c.Count != nil {
		go c.Count("get")
	}
	var t0 time.Time
	if c.Time != nil {
		t0 = time.Now()
	}

	if c.Log != nil {
		c.Log.Printf("get %q", key)
	}

	resp, ok = c.Underlying.Get(key)

	if ok {
		go c.Hit()
	}

	if c.Time != nil {
		go c.Time("get", time.Since(t0))
	}

	return
}

func (c *Async) Set(key string, data []byte) {
	if c.Count != nil {
		go c.Count("set")
	}
	var t0 time.Time
	if c.Time != nil {
		t0 = time.Now()
	}

	if c.Log != nil {
		c.Log.Printf("set %q [data: %d bytes]", key, len(data))
	}

	c.Underlying.Set(key, data)

	if c.Time != nil {
		go c.Time("set", time.Since(t0))
	}
}

func (c *Async) Delete(key string) {
	if c.Count != nil {
		go c.Count("delete")
	}
	var t0 time.Time
	if c.Time != nil {
		t0 = time.Now()
	}

	if c.Log != nil {
		c.Log.Printf("delete %q", key)
	}

	c.Underlying.Delete(key)

	if c.Time != nil {
		go c.Time("delete", time.Since(t0))
	}
}
<commit_msg>Log times if Time and Log are both set<commit_after>// Package loggedcache provides a cache wrapper that logs and records metrics
// about operations performed on an underlying cache.
package loggedcache

import (
	"log"
	"time"
)

// Async is a cache wrapper that asynchronously logs and records metrics about
// synchronous operations performed on an underlying cache.
type Async struct {
	// Underlying is the underlying cache to forward cache operations to.
	Underlying interface {
		Get(key string) (responseBytes []byte, ok bool)
		Set(key string, responseBytes []byte)
		Delete(key string)
	}

	// Count is called with "get"/"set"/"delete" for each operation performed on
	// the cache, even if it is unsuccessful.
	Count func(operation string)

	// Hit is called on each Get operation that returns data (i.e., ok is
	// true).
	Hit func()

	// Time is called with "get"/"set"/"delete" and the total duration for each
	// operation performed on the cache.
	Time func(operation string, t time.Duration)

	// Log receives log messages for operations performed on the underlying cache.
	Log *log.Logger
}

func (c *Async) Get(key string) (resp []byte, ok bool) {
	if c.Count != nil {
		go c.Count("get")
	}
	var t0 time.Time
	if c.Time != nil {
		t0 = time.Now()
	}

	if c.Log != nil {
		c.Log.Printf("get %q", key)
	}

	resp, ok = c.Underlying.Get(key)

	if ok {
		go c.Hit()
		if c.Log != nil {
			go c.Log.Printf("get hit %q", key)
		}
	} else {
		if c.Log != nil {
			go c.Log.Printf("get miss %q", key)
		}
	}

	if c.Time != nil {
		dt := time.Since(t0)
		go c.Time("get", dt)
		if c.Log != nil {
			go c.Log.Printf("get time %s [data: %d bytes]", dt, len(resp))
		}
	}

	return
}

func (c *Async) Set(key string, data []byte) {
	if c.Count != nil {
		go c.Count("set")
	}
	var t0 time.Time
	if c.Time != nil {
		t0 = time.Now()
	}

	if c.Log != nil {
		c.Log.Printf("set %q [data: %d bytes]", key, len(data))
	}

	c.Underlying.Set(key, data)

	if c.Time != nil {
		dt := time.Since(t0)
		go c.Time("set", dt)
		if c.Log != nil {
			go c.Log.Printf("set time %s [data: %d bytes]", dt, len(data))
		}
	}
}

func (c *Async) Delete(key string) {
	if c.Count != nil {
		go c.Count("delete")
	}
	var t0 time.Time
	if c.Time != nil {
		t0 = time.Now()
	}

	if c.Log != nil {
		c.Log.Printf("delete %q", key)
	}

	c.Underlying.Delete(key)

	if c.Time != nil {
		dt := time.Since(t0)
		go c.Time("delete", dt)
		if c.Log != nil {
			go c.Log.Printf("delete time %s", dt)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The go-logger Authors. All rights reserved.
// This code is MIT licensed. See the LICENSE file for more info.

package log

import (
	"bytes"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"testing"
	"runtime"
	"time"
)

func TestStream(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, os.Stdout, &buf)
	logr.Streams[1] = &buf
	if out := logr.Streams[1]; out != &buf {
		t.Errorf("Stream = %p, want %p", out, &buf)
	}
}

func TestMultiStreams(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	fPath := filepath.Join(os.TempDir(), fmt.Sprint("go_test_",
		rand.Int()))
	file, err := os.Create(fPath)
	if err != nil {
		t.Error("Create(%q) = %v; want: nil", fPath, err)
	}
	defer file.Close()
	var buf bytes.Buffer
	eLen := 55
	logr := New(LEVEL_DEBUG, file, &buf)
	logr.Debugln("Testing debug output!")
	b := make([]byte, eLen)
	n, err := file.ReadAt(b, 0)
	if n != eLen || err != nil {
		t.Errorf("Read(%d) = %d, %v; want: %d, nil", eLen, n, err,
			eLen)
	}
	if buf.Len() != eLen {
		t.Errorf("buf.Len() = %d; want: %d", buf.Len(), eLen)
	}
}

func TestLongFileFlag(t *testing.T) {
	var buf bytes.Buffer
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LlongFileName)
	SetStreams(os.Stdout, &buf)

	Debugln("Test long file flag")

	_, file, _, _ := runtime.Caller(0)

	expect := fmt.Sprintf("[DEBUG] %s: Test long file flag\n", file)

	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}

func TestShortFileFlag(t *testing.T) {
	var buf bytes.Buffer
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LshortFileName)
	SetStreams(os.Stdout, &buf)

	Debugln("Test short file flag")

	_, file, _, _ := runtime.Caller(0)

	short := file
	for i := len(file) - 1; i > 0; i-- {
		if file[i] == '/' {
			short = file[i+1:]
			break
		}
	}
	file = short

	expect := fmt.Sprintf("[DEBUG] %s: Test short file flag\n", file)

	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}

var (
	boldPrefix  = AnsiEscape(ANSI_BOLD, "TEST>", ANSI_OFF)
	colorPrefix = AnsiEscape(ANSI_BOLD, ANSI_RED, "TEST>", ANSI_OFF)
	date        = "Mon 20060102 15:04:05"
)

var outputTests = []struct {
	template   string
	prefix     string
	level      level
	dateFormat string
	flags      int
	text       string
	want       string
	wantErr    bool
}{

	// The %s format specifier is the placeholder for the date.
	{logFmt, boldPrefix, LEVEL_ALL, date, LstdFlags, "test number 1",
		"%s \x1b[1mTEST>\x1b[0m test number 1", false},

	{logFmt, colorPrefix, LEVEL_ALL, date, LstdFlags, "test number 2",
		"%s \x1b[1m\x1b[31mTEST>\x1b[0m test number 2", false},

	// Test output with coloring turned off
	{logFmt, AnsiEscape(ANSI_BOLD, "::", ANSI_OFF), LEVEL_ALL, date, Ldate,
		"test number 3", "%s :: test number 3", false},

	{logFmt, defaultPrefixColor, LEVEL_DEBUG, time.RubyDate, LstdFlags,
		"test number 4",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[37m[DEBUG]\x1b[0m test number 4",
		false},

	{logFmt, defaultPrefixColor, LEVEL_INFO, time.RubyDate, LstdFlags,
		"test number 5",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[32m[INFO]\x1b[0m test number 5",
		false},

	{logFmt, defaultPrefixColor, LEVEL_WARNING, time.RubyDate, LstdFlags,
		"test number 6",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[33m[WARNING]\x1b[0m test number 6",
		false},

	{logFmt, defaultPrefixColor, LEVEL_ERROR, time.RubyDate, LstdFlags,
		"test number 7",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[35m[ERROR]\x1b[0m test number 7",
		false},

	{logFmt, defaultPrefixColor, LEVEL_CRITICAL, time.RubyDate, LstdFlags,
		"test number 8",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[31m[CRITICAL]\x1b[0m test number 8",
		false},

	// Test date format
	{logFmt, defaultPrefixColor, LEVEL_ALL, "Mon 20060102 15:04:05",
		Ldate, "test number 9",
		"%s :: test number 9", false},
}

func TestOutput(t *testing.T) {
	for i, k := range outputTests {
		var buf bytes.Buffer
		logr := New(LEVEL_DEBUG, &buf)
		logr.Prefix = k.prefix
		logr.DateFormat = k.dateFormat
		logr.Flags = k.flags
		logr.Level = k.level
		d := time.Now().Format(logr.DateFormat)
		n, err := logr.Fprint(k.level, 1, k.text, &buf)
		if n != buf.Len() {
			t.Error("Error: ", io.ErrShortWrite)
		}
		want := fmt.Sprintf(k.want, d)
		if buf.String() != want || err != nil && !k.wantErr {
			t.Errorf("Print test %d failed, \ngot:  %q\nwant: "+
				"%q", i+1, buf.String(), want)
			continue
		}
		fmt.Printf("Test %d OK: %s\n", i, buf.String())
	}
}

func TestLevel(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, &buf)
	logr.Debug("This level should produce no output")
	if buf.Len() != 0 {
		t.Errorf("Debug() produced output at LEVEL_CRITICAL logging level")
	}
	logr.Level = LEVEL_DEBUG
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the LEVEL_DEBUG logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_CRITICAL
	logr.Println("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_ALL
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
}

func TestPrefixNewline(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(os.Stdout, &buf)
	SetLevel(LEVEL_DEBUG)
	Debugln("\n\nThis line should be padded with newlines.\n\n")

	c, err := buf.ReadString('\n')

	// If text sent with the logging functions is prepended with newlines,
	// these newlines must be prepended to the output and stripped from the
	// text. First we will make sure the two nl's are at the beginning of
	// the output.
	if c[0] != '\n' {
		t.Errorf(`First byte should be "\n", found "%s"`, string(c[0]))
	}

	c, err = buf.ReadString('\n')
	if err != nil {
		t.Error("ReadString unexpected EOF")
	}

	// Since nl should be stripped from the text and prepended to the
	// output, we must make sure the nl is still not in the middle where it
	// would be if it had not been stripped.
	nlPos := strings.Index(buf.String(), "] ") + 1
	if buf.Bytes()[nlPos+1] == '\n' {
		t.Errorf(`"\n" found at position %d.`, nlPos+1)
	}

}

func TestFlagsDate(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(os.Stdout, &buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debugln("This output should not have a date.")
	expect := "[DEBUG] This output should not have a date.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}

func TestFlagsFunctionName(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(os.Stdout, &buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LfunctionName)
	Debugln("This output should have a function name.")
	expect := "[DEBUG] TestFlagsFunction: This output should have a function name.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}
<commit_msg>logger_test.go: Remove stdout output<commit_after>// Copyright 2013 The go-logger Authors. All rights reserved.
// This code is MIT licensed. See the LICENSE file for more info.

package log

import (
	"bytes"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"testing"
	"runtime"
	"time"
)

func TestStream(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, os.Stdout, &buf)
	logr.Streams[1] = &buf
	if out := logr.Streams[1]; out != &buf {
		t.Errorf("Stream = %p, want %p", out, &buf)
	}
}

func TestMultiStreams(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	fPath := filepath.Join(os.TempDir(), fmt.Sprint("go_test_",
		rand.Int()))
	file, err := os.Create(fPath)
	if err != nil {
		t.Error("Create(%q) = %v; want: nil", fPath, err)
	}
	defer file.Close()
	var buf bytes.Buffer
	eLen := 55
	logr := New(LEVEL_DEBUG, file, &buf)
	logr.Debugln("Testing debug output!")
	b := make([]byte, eLen)
	n, err := file.ReadAt(b, 0)
	if n != eLen || err != nil {
		t.Errorf("Read(%d) = %d, %v; want: %d, nil", eLen, n, err,
			eLen)
	}
	if buf.Len() != eLen {
		t.Errorf("buf.Len() = %d; want: %d", buf.Len(), eLen)
	}
}

func TestLongFileFlag(t *testing.T) {
	var buf bytes.Buffer

	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LlongFileName)

	Debugln("Test long file flag")

	_, file, _, _ := runtime.Caller(0)

	expect := fmt.Sprintf("[DEBUG] %s: Test long file flag\n", file)

	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}

func TestShortFileFlag(t *testing.T) {
	var buf bytes.Buffer

	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LshortFileName)

	Debugln("Test short file flag")

	_, file, _, _ := runtime.Caller(0)

	short := file
	for i := len(file) - 1; i > 0; i-- {
		if file[i] == '/' {
			short = file[i+1:]
			break
		}
	}
	file = short

	expect := fmt.Sprintf("[DEBUG] %s: Test short file flag\n", file)

	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}

var (
	boldPrefix  = AnsiEscape(ANSI_BOLD, "TEST>", ANSI_OFF)
	colorPrefix = AnsiEscape(ANSI_BOLD, ANSI_RED, "TEST>", ANSI_OFF)
	date        = "Mon 20060102 15:04:05"
)

var outputTests = []struct {
	template   string
	prefix     string
	level      level
	dateFormat string
	flags      int
	text       string
	want       string
	wantErr    bool
}{

	// The %s format specifier is the placeholder for the date.
	{logFmt, boldPrefix, LEVEL_ALL, date, LstdFlags, "test number 1",
		"%s \x1b[1mTEST>\x1b[0m test number 1", false},

	{logFmt, colorPrefix, LEVEL_ALL, date, LstdFlags, "test number 2",
		"%s \x1b[1m\x1b[31mTEST>\x1b[0m test number 2", false},

	// Test output with coloring turned off
	{logFmt, AnsiEscape(ANSI_BOLD, "::", ANSI_OFF), LEVEL_ALL, date, Ldate,
		"test number 3", "%s :: test number 3", false},

	{logFmt, defaultPrefixColor, LEVEL_DEBUG, time.RubyDate, LstdFlags,
		"test number 4",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[37m[DEBUG]\x1b[0m test number 4",
		false},

	{logFmt, defaultPrefixColor, LEVEL_INFO, time.RubyDate, LstdFlags,
		"test number 5",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[32m[INFO]\x1b[0m test number 5",
		false},

	{logFmt, defaultPrefixColor, LEVEL_WARNING, time.RubyDate, LstdFlags,
		"test number 6",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[33m[WARNING]\x1b[0m test number 6",
		false},

	{logFmt, defaultPrefixColor, LEVEL_ERROR, time.RubyDate, LstdFlags,
		"test number 7",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[35m[ERROR]\x1b[0m test number 7",
		false},

	{logFmt, defaultPrefixColor, LEVEL_CRITICAL, time.RubyDate, LstdFlags,
		"test number 8",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[31m[CRITICAL]\x1b[0m test number 8",
		false},

	// Test date format
	{logFmt, defaultPrefixColor, LEVEL_ALL, "Mon 20060102 15:04:05",
		Ldate, "test number 9",
		"%s :: test number 9", false},
}

func TestOutput(t *testing.T) {
	for i, k := range outputTests {
		var buf bytes.Buffer
		logr := New(LEVEL_DEBUG, &buf)
		logr.Prefix = k.prefix
		logr.DateFormat = k.dateFormat
		logr.Flags = k.flags
		logr.Level = k.level
		d := time.Now().Format(logr.DateFormat)
		n, err := logr.Fprint(k.level, 1, k.text, &buf)
		if n != buf.Len() {
			t.Error("Error: ", io.ErrShortWrite)
		}
		want := fmt.Sprintf(k.want, d)
		if buf.String() != want || err != nil && !k.wantErr {
			t.Errorf("Print test %d failed, \ngot:  %q\nwant: "+
				"%q", i+1, buf.String(), want)
			continue
		}
	}
}

func TestLevel(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, &buf)
	logr.Debug("This level should produce no output")
	if buf.Len() != 0 {
		t.Errorf("Debug() produced output at LEVEL_CRITICAL logging level")
	}
	logr.Level = LEVEL_DEBUG
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the LEVEL_DEBUG logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_CRITICAL
	logr.Println("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_ALL
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
}

func TestPrefixNewline(t *testing.T) {
	var buf bytes.Buffer

	c, err := buf.ReadString('\n')

	// If text sent with the logging functions is prepended with newlines,
	// these newlines must be prepended to the output and stripped from the
	// text. First we will make sure the two nl's are at the beginning of
	// the output.
	if c[0] != '\n' {
		t.Errorf(`First byte should be "\n", found "%s"`, string(c[0]))
	}

	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)

	c, err = buf.ReadString('\n')
	if err != nil {
		t.Error("ReadString unexpected EOF")
	}

	// Since nl should be stripped from the text and prepended to the
	// output, we must make sure the nl is still not in the middle where it
	// would be if it had not been stripped.
	nlPos := strings.Index(buf.String(), "] ") + 1
	if buf.Bytes()[nlPos+1] == '\n' {
		t.Errorf(`"\n" found at position %d.`, nlPos+1)
	}
}

func TestFlagsDate(t *testing.T) {
	var buf bytes.Buffer

	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)

	Debugln("This output should not have a date.")

	expect := "[DEBUG] This output should not have a date.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}

func TestFlagsFunctionName(t *testing.T) {
	var buf bytes.Buffer

	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LfunctionName)

	Debugln("This output should have a function name.")
	expect := "[DEBUG] TestFlagsFunction: This output should have a function name.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%s\nGot:\n\t%s\n", expect, buf.String())
	}
}
<|endoftext|>


<commit_before>package coreapi

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"

	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
	dagutils "github.com/ipfs/go-ipfs/merkledag/utils"

	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	dag "github.com/ipfs/go-ipfs/merkledag"
	ft "github.com/ipfs/go-ipfs/unixfs"

	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	node "gx/ipfs/QmNwUEK7QbwSqyKBu3mMtToo8SUc6wQJ7gdZq4gGGJqfnf/go-ipld-format"
	cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid"
)

const inputLimit = 2 << 20

type ObjectAPI struct {
	*CoreAPI
	*caopts.ObjectOptions
}

type Link struct {
	Name, Hash string
	Size       uint64
}

type Node struct {
	Links []Link
	Data  string
}

func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (coreiface.Node, error) {
	options, err := caopts.ObjectNewOptions(opts...)
	if err != nil {
		return nil, err
	}

	var n node.Node
	switch options.Type {
	case "empty":
		n = new(dag.ProtoNode)
	case "unixfs-dir":
		n = ft.EmptyDirNode()
	}

	_, err = api.node.DAG.Add(n)
	if err != nil {
		return nil, err
	}
	return n, nil
}

func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.ObjectPutOption) (coreiface.Path, error) {
	options, err := caopts.ObjectPutOptions(opts...)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(io.LimitReader(src, inputLimit+10))
	if err != nil {
		return nil, err
	}

	var dagnode *dag.ProtoNode
	switch options.InputEnc {
	case "json":
		node := new(Node)
		err = json.Unmarshal(data, node)
		if err != nil {
			return nil, err
		}

		// check that we have data in the Node to add
		// otherwise we will add the empty object without raising an error
		if nodeEmpty(node) {
			return nil, errors.New("no data or links in this node")
		}

		dagnode, err = deserializeNode(node, options.DataType)
		if err != nil {
			return nil, err
		}

	case "protobuf":
		dagnode, err = dag.DecodeProtobuf(data)

	case "xml":
		node := new(Node)
		err = xml.Unmarshal(data, node)
		if err != nil {
			return nil, err
		}

		// check that we have data in the Node to add
		// otherwise we will add the empty object without raising an error
		if nodeEmpty(node) {
			return nil, errors.New("no data or links in this node")
		}

		dagnode, err = deserializeNode(node, options.DataType)
		if err != nil {
			return nil, err
		}

	default:
		return nil, errors.New("unknown object encoding")
	}

	if err != nil {
		return nil, err
	}

	_, err = api.node.DAG.Add(dagnode)
	if err != nil {
		return nil, err
	}

	return ParseCid(dagnode.Cid()), nil
}

func (api *ObjectAPI) Get(ctx context.Context, path coreiface.Path) (coreiface.Node, error) {
	return api.core().ResolveNode(ctx, path)
}

func (api *ObjectAPI) Data(ctx context.Context, path coreiface.Path) (io.Reader, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	pbnd, ok := nd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	return bytes.NewReader(pbnd.Data()), nil
}

func (api *ObjectAPI) Links(ctx context.Context, path coreiface.Path) ([]*coreiface.Link, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	links := nd.Links()
	out := make([]*coreiface.Link, len(links))
	for n, l := range links {
		out[n] = (*coreiface.Link)(l)
	}

	return out, nil
}

func (api *ObjectAPI) Stat(ctx context.Context, path coreiface.Path) (*coreiface.ObjectStat, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	stat, err := nd.Stat()
	if err != nil {
		return nil, err
	}

	out := &coreiface.ObjectStat{
		Cid:            nd.Cid(),
		NumLinks:       stat.NumLinks,
		BlockSize:      stat.BlockSize,
		LinksSize:      stat.LinksSize,
		DataSize:       stat.DataSize,
		CumulativeSize: stat.CumulativeSize,
	}

	return out, nil
}

func (api *ObjectAPI) AddLink(ctx context.Context, base coreiface.Path, name string, child coreiface.Path, opts ...caopts.ObjectAddLinkOption) (coreiface.Path, error) {
	options, err := caopts.ObjectAddLinkOptions(opts...)
	if err != nil {
		return nil, err
	}

	baseNd, err := api.core().ResolveNode(ctx, base)
	if err != nil {
		return nil, err
	}

	childNd, err := api.core().ResolveNode(ctx, child)
	if err != nil {
		return nil, err
	}

	basePb, ok := baseNd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	var createfunc func() *dag.ProtoNode
	if options.Create {
		createfunc = ft.EmptyDirNode
	}

	e := dagutils.NewDagEditor(basePb, api.node.DAG)

	err = e.InsertNodeAtPath(ctx, name, childNd, createfunc)
	if err != nil {
		return nil, err
	}

	nnode, err := e.Finalize(api.node.DAG)
	if err != nil {
		return nil, err
	}

	return ParseCid(nnode.Cid()), nil
}

func (api *ObjectAPI) RmLink(ctx context.Context, base coreiface.Path, link string) (coreiface.Path, error) {
	baseNd, err := api.core().ResolveNode(ctx, base)
	if err != nil {
		return nil, err
	}

	basePb, ok := baseNd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	e := dagutils.NewDagEditor(basePb, api.node.DAG)

	err = e.RmLink(ctx, link)
	if err != nil {
		return nil, err
	}

	nnode, err := e.Finalize(api.node.DAG)
	if err != nil {
		return nil, err
	}

	return ParseCid(nnode.Cid()), nil
}

func (api *ObjectAPI) AppendData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Path, error) {
	return api.patchData(ctx, path, r, true)
}

func (api *ObjectAPI) SetData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Path, error) {
	return api.patchData(ctx, path, r, false)
}

func (api *ObjectAPI) patchData(ctx context.Context, path coreiface.Path, r io.Reader, appendData bool) (coreiface.Path, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	pbnd, ok := nd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	if appendData {
		data = append(pbnd.Data(), data...)
	}
	pbnd.SetData(data)

	_, err = api.node.DAG.Add(pbnd)
	if err != nil {
		return nil, err
	}

	return ParseCid(pbnd.Cid()), nil
}

func (api *ObjectAPI) core() coreiface.CoreAPI {
	return api.CoreAPI
}

func deserializeNode(nd *Node, dataFieldEncoding string) (*dag.ProtoNode, error) {
	dagnode := new(dag.ProtoNode)
	switch dataFieldEncoding {
	case "text":
		dagnode.SetData([]byte(nd.Data))
	case "base64":
		data, _ := base64.StdEncoding.DecodeString(nd.Data)
		dagnode.SetData(data)
	default:
		return nil, fmt.Errorf("Unkown data field encoding")
	}

	dagnode.SetLinks(make([]*node.Link, len(nd.Links)))
	for i, link := range nd.Links {
		c, err := cid.Decode(link.Hash)
		if err != nil {
			return nil, err
		}
		dagnode.Links()[i] = &node.Link{
			Name: link.Name,
			Size: link.Size,
			Cid:  c,
		}
	}

	return dagnode, nil
}

func nodeEmpty(node *Node) bool {
	return node.Data == "" && len(node.Links) == 0
}
<commit_msg>fix hidden merge conflicts<commit_after>package coreapi

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"

	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
	dagutils "github.com/ipfs/go-ipfs/merkledag/utils"

	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	dag "github.com/ipfs/go-ipfs/merkledag"
	ft "github.com/ipfs/go-ipfs/unixfs"

	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
	node "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)

const inputLimit = 2 << 20

type ObjectAPI struct {
	*CoreAPI
	*caopts.ObjectOptions
}

type Link struct {
	Name, Hash string
	Size       uint64
}

type Node struct {
	Links []Link
	Data  string
}

func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (coreiface.Node, error) {
	options, err := caopts.ObjectNewOptions(opts...)
	if err != nil {
		return nil, err
	}

	var n node.Node
	switch options.Type {
	case "empty":
		n = new(dag.ProtoNode)
	case "unixfs-dir":
		n = ft.EmptyDirNode()
	}

	err = api.node.DAG.Add(ctx, n)
	if err != nil {
		return nil, err
	}
	return n, nil
}

func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.ObjectPutOption) (coreiface.Path, error) {
	options, err := caopts.ObjectPutOptions(opts...)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(io.LimitReader(src, inputLimit+10))
	if err != nil {
		return nil, err
	}

	var dagnode *dag.ProtoNode
	switch options.InputEnc {
	case "json":
		node := new(Node)
		err = json.Unmarshal(data, node)
		if err != nil {
			return nil, err
		}

		// check that we have data in the Node to add
		// otherwise we will add the empty object without raising an error
		if nodeEmpty(node) {
			return nil, errors.New("no data or links in this node")
		}

		dagnode, err = deserializeNode(node, options.DataType)
		if err != nil {
			return nil, err
		}

	case "protobuf":
		dagnode, err = dag.DecodeProtobuf(data)

	case "xml":
		node := new(Node)
		err = xml.Unmarshal(data, node)
		if err != nil {
			return nil, err
		}

		// check that we have data in the Node to add
		// otherwise we will add the empty object without raising an error
		if nodeEmpty(node) {
			return nil, errors.New("no data or links in this node")
		}

		dagnode, err = deserializeNode(node, options.DataType)
		if err != nil {
			return nil, err
		}

	default:
		return nil, errors.New("unknown object encoding")
	}

	if err != nil {
		return nil, err
	}

	err = api.node.DAG.Add(ctx, dagnode)
	if err != nil {
		return nil, err
	}

	return ParseCid(dagnode.Cid()), nil
}

func (api *ObjectAPI) Get(ctx context.Context, path coreiface.Path) (coreiface.Node, error) {
	return api.core().ResolveNode(ctx, path)
}

func (api *ObjectAPI) Data(ctx context.Context, path coreiface.Path) (io.Reader, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	pbnd, ok := nd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	return bytes.NewReader(pbnd.Data()), nil
}

func (api *ObjectAPI) Links(ctx context.Context, path coreiface.Path) ([]*coreiface.Link, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	links := nd.Links()
	out := make([]*coreiface.Link, len(links))
	for n, l := range links {
		out[n] = (*coreiface.Link)(l)
	}

	return out, nil
}

func (api *ObjectAPI) Stat(ctx context.Context, path coreiface.Path) (*coreiface.ObjectStat, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	stat, err := nd.Stat()
	if err != nil {
		return nil, err
	}

	out := &coreiface.ObjectStat{
		Cid:            nd.Cid(),
		NumLinks:       stat.NumLinks,
		BlockSize:      stat.BlockSize,
		LinksSize:      stat.LinksSize,
		DataSize:       stat.DataSize,
		CumulativeSize: stat.CumulativeSize,
	}

	return out, nil
}

func (api *ObjectAPI) AddLink(ctx context.Context, base coreiface.Path, name string, child coreiface.Path, opts ...caopts.ObjectAddLinkOption) (coreiface.Path, error) {
	options, err := caopts.ObjectAddLinkOptions(opts...)
	if err != nil {
		return nil, err
	}

	baseNd, err := api.core().ResolveNode(ctx, base)
	if err != nil {
		return nil, err
	}

	childNd, err := api.core().ResolveNode(ctx, child)
	if err != nil {
		return nil, err
	}

	basePb, ok := baseNd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	var createfunc func() *dag.ProtoNode
	if options.Create {
		createfunc = ft.EmptyDirNode
	}

	e := dagutils.NewDagEditor(basePb, api.node.DAG)

	err = e.InsertNodeAtPath(ctx, name, childNd, createfunc)
	if err != nil {
		return nil, err
	}

	nnode, err := e.Finalize(ctx, api.node.DAG)
	if err != nil {
		return nil, err
	}

	return ParseCid(nnode.Cid()), nil
}

func (api *ObjectAPI) RmLink(ctx context.Context, base coreiface.Path, link string) (coreiface.Path, error) {
	baseNd, err := api.core().ResolveNode(ctx, base)
	if err != nil {
		return nil, err
	}

	basePb, ok := baseNd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	e := dagutils.NewDagEditor(basePb, api.node.DAG)

	err = e.RmLink(ctx, link)
	if err != nil {
		return nil, err
	}

	nnode, err := e.Finalize(ctx, api.node.DAG)
	if err != nil {
		return nil, err
	}

	return ParseCid(nnode.Cid()), nil
}

func (api *ObjectAPI) AppendData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Path, error) {
	return api.patchData(ctx, path, r, true)
}

func (api *ObjectAPI) SetData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Path, error) {
	return api.patchData(ctx, path, r, false)
}

func (api *ObjectAPI) patchData(ctx context.Context, path coreiface.Path, r io.Reader, appendData bool) (coreiface.Path, error) {
	nd, err := api.core().ResolveNode(ctx, path)
	if err != nil {
		return nil, err
	}

	pbnd, ok := nd.(*dag.ProtoNode)
	if !ok {
		return nil, dag.ErrNotProtobuf
	}

	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	if appendData {
		data = append(pbnd.Data(), data...)
	}
	pbnd.SetData(data)

	err = api.node.DAG.Add(ctx, pbnd)
	if err != nil {
		return nil, err
	}

	return ParseCid(pbnd.Cid()), nil
}

func (api *ObjectAPI) core() coreiface.CoreAPI {
	return api.CoreAPI
}

func deserializeNode(nd *Node, dataFieldEncoding string) (*dag.ProtoNode, error) {
	dagnode := new(dag.ProtoNode)
	switch dataFieldEncoding {
	case "text":
		dagnode.SetData([]byte(nd.Data))
	case "base64":
		data, _ := base64.StdEncoding.DecodeString(nd.Data)
		dagnode.SetData(data)
	default:
		return nil, fmt.Errorf("Unkown data field encoding")
	}

	dagnode.SetLinks(make([]*node.Link, len(nd.Links)))
	for i, link := range nd.Links {
		c, err := cid.Decode(link.Hash)
		if err != nil {
			return nil, err
		}
		dagnode.Links()[i] = &node.Link{
			Name: link.Name,
			Size: link.Size,
			Cid:  c,
		}
	}

	return dagnode, nil
}

func nodeEmpty(node *Node) bool {
	return node.Data == "" && len(node.Links) == 0
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
	"github.com/koding/kloud/kloud"
)

// Kloud represents a remote kloud instance
type Kloud struct {
	client *kite.Client
}

// Kloud returns a new connected kloud instance. The kloud is ready to use.
// It's connected and will redial if there is any disconnections.
func NewKloud(k *kite.Kite) (*Kloud, error) {
	kontrolQuery := &protocol.KontrolQuery{
		Username:    "koding",
		Environment: "vagrant",
		Name:        "kloud",
	}

	timeout := time.After(time.Minute)

	k.Log.Info("Querying for Kloud: %+v", kontrolQuery)
	for {
		select {
		case <-time.Tick(time.Second * 2):
			kites, err := k.GetKites(kontrolQuery)
			if err != nil {
				// still not up, try again until the kite is ready
				continue
			}

			remoteKite := kites[0]

			connected, err := remoteKite.DialForever()
			if err != nil {
				return nil, err
			}

			select {
			case <-connected:
			case <-time.After(time.Minute):
				return nil, kloud.NewError(kloud.ErrNoKiteConnection)
			}

			// Kloud connection is ready now
			return &Kloud{
				client: remoteKite,
			}, nil
		case <-timeout:
			return nil, fmt.Errorf("timeout while connection for kite")
		}
	}
}

// Report reports machine and usage metrics to kloud instance.
func (k *Kloud) Report() error {
	// update information before we send it
	usg.Update()

	fmt.Printf("repoting usage %+v\n", usg)
	_, err := k.client.Tell("report", usg)
	return err
}
<commit_msg>kloud: move github.com/koding/kloud to koding/kites/kloud/kloud<commit_after>package main

import (
	"fmt"
	"time"

	"koding/kites/kloud/kloud"

	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
)

// Kloud represents a remote kloud instance
type Kloud struct {
	client *kite.Client
}

// Kloud returns a new connected kloud instance. The kloud is ready to use.
// It's connected and will redial if there is any disconnections.
func NewKloud(k *kite.Kite) (*Kloud, error) {
	kontrolQuery := &protocol.KontrolQuery{
		Username:    "koding",
		Environment: "vagrant",
		Name:        "kloud",
	}

	timeout := time.After(time.Minute)

	k.Log.Info("Querying for Kloud: %+v", kontrolQuery)
	for {
		select {
		case <-time.Tick(time.Second * 2):
			kites, err := k.GetKites(kontrolQuery)
			if err != nil {
				// still not up, try again until the kite is ready
				continue
			}

			remoteKite := kites[0]

			connected, err := remoteKite.DialForever()
			if err != nil {
				return nil, err
			}

			select {
			case <-connected:
			case <-time.After(time.Minute):
				return nil, kloud.NewError(kloud.ErrNoKiteConnection)
			}

			// Kloud connection is ready now
			return &Kloud{
				client: remoteKite,
			}, nil
		case <-timeout:
			return nil, fmt.Errorf("timeout while connection for kite")
		}
	}
}

// Report reports machine and usage metrics to kloud instance.
func (k *Kloud) Report() error {
	// update information before we send it
	usg.Update()

	fmt.Printf("repoting usage %+v\n", usg)
	_, err := k.client.Tell("report", usg)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	"bytes"
	"math"

	"github.com/juju/errors"
)

// Key represents high-level Key type.
type Key []byte

// Next returns the next key in byte-order.
func (k Key) Next() Key {
	// add 0x0 to the end of key
	buf := make([]byte, len([]byte(k))+1)
	copy(buf, []byte(k))
	return buf
}

// Cmp returns the comparison result of two key.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (k Key) Cmp(another Key) int {
	return bytes.Compare(k, another)
}

// EncodedKey represents encoded key in low-level storage engine.
type EncodedKey []byte

// Cmp returns the comparison result of two key.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (k EncodedKey) Cmp(another EncodedKey) int {
	return bytes.Compare(k, another)
}

// Next returns the next key in byte-order.
func (k EncodedKey) Next() EncodedKey {
	return EncodedKey(bytes.Join([][]byte{k, Key{0}}, nil))
}

// VersionProvider provides increasing IDs.
type VersionProvider interface {
	CurrentVersion() (Version, error)
}

// Version is the wrapper of KV's version.
type Version struct {
	Ver uint64
}

var (
	// MaxVersion is the maximum version, notice that it's not a valid version.
	MaxVersion = Version{Ver: math.MaxUint64}
	// MinVersion is the minimum version, it's not a valid version, too.
	MinVersion = Version{Ver: 0}
)

// NewVersion creates a new Version struct.
func NewVersion(v uint64) Version {
	return Version{
		Ver: v,
	}
}

// Cmp returns the comparison result of two versions.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (v Version) Cmp(another Version) int {
	if v.Ver > another.Ver {
		return 1
	} else if v.Ver < another.Ver {
		return -1
	}
	return 0
}

// DecodeFn is a function that decodes data after fetching from store.
type DecodeFn func(raw interface{}) (interface{}, error)

// EncodeFn is a function that encodes data before putting into store.
type EncodeFn func(raw interface{}) (interface{}, error)

// ErrNotCommitted is the error returned by CommitVersion when this
// transaction is not committed.
var ErrNotCommitted = errors.New("this transaction is not committed")

// Option is used for customizing kv store's behaviors during a transaction.
type Option int

// Options is an interface of a set of options. Each option is associated with a value.
type Options interface {
	// Get gets an option value.
	Get(opt Option) (v interface{}, ok bool)
}

const (
	// RangePrefetchOnCacheMiss directives that when dealing with a Get operation but failing to read data from cache,
	// it will launch a RangePrefetch to underlying storage instead of Get. The range starts from requested key and
	// has a limit of the option value. The feature is disabled if option value <= 0 or value type is not int.
	// This option is particularly useful when we have to do sequential Gets, e.g. table scans.
	RangePrefetchOnCacheMiss Option = iota + 1

	// PresumeKeyNotExists directives that when dealing with a Get operation but failing to read data from cache,
	// we presume that the key does not exist in Store. The actual existence will be checked before the
	// transaction's commit.
	// This option is an optimization for frequent checks during a transaction, e.g. batch inserts.
	PresumeKeyNotExists
)

// Retriever is the interface wraps the basic Get and Seek methods.
type Retriever interface {
	// Get gets the value for key k from KV storage.
	Get(k Key) ([]byte, error)
	// Seek searches for the entry with key k in KV storage.
	Seek(k Key) (Iterator, error)
}

// Mutator is the interface wraps the basic Set and Delete methods.
type Mutator interface {
	// Set sets the value for key k as v into KV storage.
	Set(k Key, v []byte) error
	// Delete removes the entry for key k from KV storage.
	Delete(k Key) error
}

// RetrieverMutator is the interface that groups Retriever and Mutator interface.
type RetrieverMutator interface {
	Retriever
	Mutator
}

// UnionStore is an in-memory Store which contains a buffer for write and a
// snapshot for read.
type UnionStore interface {
	RetrieverMutator
	// Inc increases the value for key k in KV storage by step.
	Inc(k Key, step int64) (int64, error)
	// GetInt64 get int64 which created by Inc method.
	GetInt64(k Key) (int64, error)
	// CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched.
	// Lazy condition pairs should be checked before transaction commit.
	CheckLazyConditionPairs() error
	// WalkWriteBuffer iterates all buffered write kv pairs.
	WalkWriteBuffer(f func(Iterator) error) error
	// BatchPrefetch fetches values from KV storage to cache for later use.
	BatchPrefetch(keys []Key) error
	// RangePrefetch fetches values in the range [start, end] from KV storage
	// to cache for later use. Maximum number of values is up to limit.
	RangePrefetch(start, end Key, limit int) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// ReleaseSnapshot releases underlying snapshot.
	ReleaseSnapshot()
	// Close releases all resources.
	Close()
}

// Transaction defines the interface for operations inside a Transaction.
// This is not thread safe.
type Transaction interface {
	UnionStore
	// Commit commits the transaction operations to KV store.
	Commit() error
	// CommittedVersion returns the version of this committed transaction. If this
	// transaction has not been committed, returns ErrNotCommitted error.
	CommittedVersion() (Version, error)
	// Rollback undoes the transaction operations to KV store.
	Rollback() error
	// String implements fmt.Stringer interface.
	String() string
	// LockKeys tries to lock the entries with the keys in KV store.
	LockKeys(keys ...Key) error
}

// MvccSnapshot is used to get/seek a specific version in a snapshot.
type MvccSnapshot interface {
	// MvccGet returns the specific version of given key, if the version doesn't
	// exist, returns the nearest(lower) version's data.
	MvccGet(k Key, ver Version) ([]byte, error)
	// MvccIterator seeks to the key in the specific version's snapshot, if the
	// version doesn't exist, returns the nearest(lower) version's snaphot.
	NewMvccIterator(k Key, ver Version) Iterator
	// Release releases this snapshot.
	MvccRelease()
}

// Snapshot defines the interface for the snapshot fetched from KV store.
type Snapshot interface {
	Retriever
	// BatchGet gets a batch of values from snapshot.
	BatchGet(keys []Key) (map[string][]byte, error)
	// RangeGet gets values in the range [start, end] from snapshot. Maximum
	// number of values is up to limit.
	RangeGet(start, end Key, limit int) (map[string][]byte, error)
	// Release releases the snapshot to store.
	Release()
}

// MemBuffer is the interface for transaction buffer of update in a transaction
type MemBuffer interface {
	RetrieverMutator
	// Release releases the buffer.
	Release()
}

// Driver is the interface that must be implemented by a KV storage.
type Driver interface {
	// Open returns a new Storage.
	// The schema is the string for storage specific format.
	Open(schema string) (Storage, error)
}

// Storage defines the interface for storage.
// Isolation should be at least SI(SNAPSHOT ISOLATION)
type Storage interface {
	// Begin transaction
	Begin() (Transaction, error)
	// GetSnapshot gets a snaphot that is able to read any data which data is <= ver.
	// if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
	GetSnapshot(ver Version) (MvccSnapshot, error)
	// Close store
	Close() error
	// Storage's unique ID
	UUID() string
	// CurrentVersion returns current max committed version.
	CurrentVersion() (Version, error)
}

// FnKeyCmp is the function for iterator the keys
type FnKeyCmp func(key Key) bool

// Iterator is the interface for a interator on KV store.
type Iterator interface {
	Next() error
	Value() []byte
	Key() string
	Valid() bool
	Close()
}

// IndexIterator is the interface for iterator of index data on KV store.
type IndexIterator interface {
	Next() (k []interface{}, h int64, err error)
	Close()
}

// Index is the interface for index data on KV store.
type Index interface {
	Create(txn Transaction, indexedValues []interface{}, h int64) error                          // supports insert into statement
	Delete(txn Transaction, indexedValues []interface{}, h int64) error                          // supports delete from statement
	Drop(txn Transaction) error                                                                  // supports drop table, drop index statements
	Exist(txn Transaction, indexedValues []interface{}, h int64) (bool, int64, error)            // supports check index exist
	GenIndexKey(indexedValues []interface{}, h int64) (key []byte, distinct bool, err error)     // supports index check
	Seek(txn Transaction, indexedValues []interface{}) (iter IndexIterator, hit bool, err error) // supports where clause
	SeekFirst(txn Transaction) (iter IndexIterator, err error)                                   // supports aggregate min / ascending order by
}
<commit_msg>kv: update comments<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	"bytes"
	"math"

	"github.com/juju/errors"
)

// Key represents high-level Key type.
type Key []byte

// Next returns the next key in byte-order.
func (k Key) Next() Key {
	// add 0x0 to the end of key
	buf := make([]byte, len([]byte(k))+1)
	copy(buf, []byte(k))
	return buf
}

// Cmp returns the comparison result of two key.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (k Key) Cmp(another Key) int {
	return bytes.Compare(k, another)
}

// EncodedKey represents encoded key in low-level storage engine.
type EncodedKey []byte

// Cmp returns the comparison result of two key.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (k EncodedKey) Cmp(another EncodedKey) int {
	return bytes.Compare(k, another)
}

// Next returns the next key in byte-order.
func (k EncodedKey) Next() EncodedKey {
	return EncodedKey(bytes.Join([][]byte{k, Key{0}}, nil))
}

// VersionProvider provides increasing IDs.
type VersionProvider interface {
	CurrentVersion() (Version, error)
}

// Version is the wrapper of KV's version.
type Version struct {
	Ver uint64
}

var (
	// MaxVersion is the maximum version, notice that it's not a valid version.
	MaxVersion = Version{Ver: math.MaxUint64}
	// MinVersion is the minimum version, it's not a valid version, too.
	MinVersion = Version{Ver: 0}
)

// NewVersion creates a new Version struct.
func NewVersion(v uint64) Version {
	return Version{
		Ver: v,
	}
}

// Cmp returns the comparison result of two versions.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (v Version) Cmp(another Version) int {
	if v.Ver > another.Ver {
		return 1
	} else if v.Ver < another.Ver {
		return -1
	}
	return 0
}

// DecodeFn is a function that decodes data after fetching from store.
type DecodeFn func(raw interface{}) (interface{}, error)

// EncodeFn is a function that encodes data before putting into store.
type EncodeFn func(raw interface{}) (interface{}, error)

// ErrNotCommitted is the error returned by CommitVersion when this
// transaction is not committed.
var ErrNotCommitted = errors.New("this transaction is not committed")

// Option is used for customizing kv store's behaviors during a transaction.
type Option int

// Options is an interface of a set of options. Each option is associated with a value.
type Options interface {
	// Get gets an option value.
	Get(opt Option) (v interface{}, ok bool)
}

const (
	// RangePrefetchOnCacheMiss directives that when dealing with a Get operation but failing to read data from cache,
	// it will launch a RangePrefetch to underlying storage instead of Get. The range starts from requested key and
	// has a limit of the option value. The feature is disabled if option value <= 0 or value type is not int.
	// This option is particularly useful when we have to do sequential Gets, e.g. table scans.
	RangePrefetchOnCacheMiss Option = iota + 1

	// PresumeKeyNotExists directives that when dealing with a Get operation but failing to read data from cache,
	// we presume that the key does not exist in Store. The actual existence will be checked before the
	// transaction's commit.
	// This option is an optimization for frequent checks during a transaction, e.g. batch inserts.
	PresumeKeyNotExists
)

// Retriever is the interface wraps the basic Get and Seek methods.
type Retriever interface {
	// Get gets the value for key k from KV storage.
	Get(k Key) ([]byte, error)
	// Seek searches for the entry with key k in KV storage.
	Seek(k Key) (Iterator, error)
}

// Mutator is the interface wraps the basic Set and Delete methods.
type Mutator interface {
	// Set sets the value for key k as v into KV storage.
	Set(k Key, v []byte) error
	// Delete removes the entry for key k from KV storage.
	Delete(k Key) error
}

// RetrieverMutator is the interface that groups Retriever and Mutator interface.
type RetrieverMutator interface {
	Retriever
	Mutator
}

// UnionStore is an in-memory Store which contains a buffer for write and a
// snapshot for read.
type UnionStore interface {
	RetrieverMutator
	// Inc increases the value for key k in KV storage by step.
	Inc(k Key, step int64) (int64, error)
	// GetInt64 get int64 which created by Inc method.
	GetInt64(k Key) (int64, error)
	// CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched.
	// Lazy condition pairs should be checked before transaction commit.
	CheckLazyConditionPairs() error
	// WalkWriteBuffer iterates all buffered write kv pairs.
	WalkWriteBuffer(f func(Iterator) error) error
	// BatchPrefetch fetches values from KV storage to cache for later use.
	BatchPrefetch(keys []Key) error
	// RangePrefetch fetches values in the range [start, end] from KV storage
	// to cache for later use. Maximum number of values is up to limit.
	RangePrefetch(start, end Key, limit int) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// ReleaseSnapshot releases underlying snapshot.
	ReleaseSnapshot()
	// Close releases all resources.
	Close()
}

// Transaction defines the interface for operations inside a Transaction.
// This is not thread safe.
type Transaction interface {
	UnionStore
	// Commit commits the transaction operations to KV store.
	Commit() error
	// CommittedVersion returns the version of this committed transaction. If this
	// transaction has not been committed, returns ErrNotCommitted error.
	CommittedVersion() (Version, error)
	// Rollback undoes the transaction operations to KV store.
	Rollback() error
	// String implements fmt.Stringer interface.
	String() string
	// LockKeys tries to lock the entries with the keys in KV store.
	LockKeys(keys ...Key) error
}

// MvccSnapshot is used to get/seek a specific version in a snapshot.
type MvccSnapshot interface {
	// MvccGet returns the specific version of given key, if the version doesn't
	// exist, returns the nearest(lower) version's data.
	MvccGet(k Key, ver Version) ([]byte, error)
	// MvccIterator seeks to the key in the specific version's snapshot, if the
	// version doesn't exist, returns the nearest(lower) version's snaphot.
	NewMvccIterator(k Key, ver Version) Iterator
	// Release releases this snapshot.
	MvccRelease()
}

// Snapshot defines the interface for the snapshot fetched from KV store.
type Snapshot interface {
	Retriever
	// BatchGet gets a batch of values from snapshot.
	BatchGet(keys []Key) (map[string][]byte, error)
	// RangeGet gets values in the range [start, end] from snapshot. Maximum
	// number of values is up to limit.
	RangeGet(start, end Key, limit int) (map[string][]byte, error)
	// Release releases the snapshot to store.
	Release()
}

// MemBuffer is the interface for transaction buffer of update in a transaction
type MemBuffer interface {
	RetrieverMutator
	// Release releases the buffer.
	Release()
}

// Driver is the interface that must be implemented by a KV storage.
type Driver interface {
	// Open returns a new Storage.
	// The schema is the string for storage specific format.
	Open(schema string) (Storage, error)
}

// Storage defines the interface for storage.
// Isolation should be at least SI(SNAPSHOT ISOLATION)
type Storage interface {
	// Begin transaction
	Begin() (Transaction, error)
	// GetSnapshot gets a snaphot that is able to read any data which data is <= ver.
	// if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
	GetSnapshot(ver Version) (MvccSnapshot, error)
	// Close store
	Close() error
	// Storage's unique ID
	UUID() string
	// CurrentVersion returns current max committed version.
	CurrentVersion() (Version, error)
}

// FnKeyCmp is the function for iterator the keys
type FnKeyCmp func(key Key) bool

// Iterator is the interface for a interator on KV store.
type Iterator interface {
	Next() error
	Value() []byte
	Key() string
	Valid() bool
	Close()
}

// IndexIterator is the interface for iterator of index data on KV store.
type IndexIterator interface {
	Next() (k []interface{}, h int64, err error)
	Close()
}

// Index is the interface for index data on KV store.
type Index interface {
	// Create supports insert into statement.
	Create(txn Transaction, indexedValues []interface{}, h int64) error
	// Delete supports delete from statement.
	Delete(txn Transaction, indexedValues []interface{}, h int64) error
	// Drop supports drop table, drop index statements.
	Drop(txn Transaction) error
	// Exist supports check index exists or not.
	Exist(txn Transaction, indexedValues []interface{}, h int64) (bool, int64, error)
	// GenIndexKey generates an index key.
	GenIndexKey(indexedValues []interface{}, h int64) (key []byte, distinct bool, err error)
	// Seek supports where clause.
	Seek(txn Transaction, indexedValues []interface{}) (iter IndexIterator, hit bool, err error)
	// SeekFirst supports aggregate min and ascend order by.
	SeekFirst(txn Transaction) (iter IndexIterator, err error)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package dig

import (
	"fmt"
	"reflect"
)

// The param interface represents a dependency for a constructor.
//
// The following implementations exist:
//  paramList     All arguments of the constructor.
//  paramSingle   An explicitly requested type.
//  paramObject   dig.In struct where each field in the struct can be another
//                param.
type param interface {
	fmt.Stringer

	// Comprehensive list of dependencies this parameter represents.
	Dependencies() []edge
}

var (
	_ param = paramSingle{}
	_ param = paramObject{}
	_ param = paramList{}
)

// Calls the provided function on all paramSingles in the given param tree.
func forEachParamSingle(param param, f func(paramSingle)) {
	switch p := param.(type) {
	case paramList:
		for _, arg := range p.Params {
			forEachParamSingle(arg, f)
		}
	case paramSingle:
		f(p)
	case paramObject:
		for _, field := range p.Fields {
			forEachParamSingle(field.Param, f)
		}
	default:
		panic(fmt.Sprintf(
			"It looks like you have found a bug in dig. "+
				"Please file an issue at https://github.com/uber-go/dig/issues/ "+
				"and provide the following message: "+
				"received unknown param type %T", param))
	}
}

// newParam builds a param from the given type. If the provided type is a
// dig.In struct, an paramObject will be returned.
func newParam(t reflect.Type) (param, error) {
	if IsIn(t) {
		return newParamObject(t)
	}
	return paramSingle{Type: t}, nil
}

// paramList holds all arguments of the constructor as params.
type paramList struct {
	ctype reflect.Type // type of the constructor

	Params []param
}

// newParamList builds a paramList from the provided constructor type.
//
// Variadic arguments of a constructor are ignored and not included as
// dependencies.
func newParamList(ctype reflect.Type) (paramList, error) {
	numArgs := ctype.NumIn()
	if ctype.IsVariadic() {
		// NOTE: If the function is variadic, we skip the last argument
		// because we're not filling variadic arguments yet. See #120.
		numArgs--
	}

	pl := paramList{
		ctype:  ctype,
		Params: make([]param, 0, numArgs),
	}

	for i := 0; i < numArgs; i++ {
		p, err := newParam(ctype.In(i))
		if err != nil {
			return pl, errWrapf(err, "bad argument %d", i+1)
		}
		pl.Params = append(pl.Params, p)
	}
	return pl, nil
}

func (pl paramList) Dependencies() []edge {
	var deps []edge
	for _, p := range pl.Params {
		deps = append(deps, p.Dependencies()...)
	}
	return deps
}

// paramSingle is an explicitly requested type, optionally with a name.
//
// This object must be present in the graph as-is unless it's specified as
// optional.
type paramSingle struct {
	Name     string
	Optional bool
	Type     reflect.Type
}

func (ps paramSingle) Dependencies() []edge {
	return []edge{
		{key: key{t: ps.Type, name: ps.Name}, optional: ps.Optional},
	}
}

// paramObjectField is a single field of a dig.In struct.
type paramObjectField struct {
	// Name of the field in the struct.
	FieldName string

	// Index of this field in the target struct.
	//
	// We need to track this separately because not all fields of the
	// struct map to params.
	FieldIndex int

	// The dependency requested by this field.
	Param param
}

// paramObject is a dig.In struct where each field is another param.
//
// This object is not expected in the graph as-is.
type paramObject struct {
	Type   reflect.Type
	Fields []paramObjectField

	deps []edge
}

// newParamObject builds an paramObject from the provided type. The type MUST
// be a dig.In struct.
func newParamObject(t reflect.Type) (paramObject, error) {
	po := paramObject{Type: t}

	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		if f.Type == _inType {
			// Skip over the dig.In embed.
			continue
		}

		if f.PkgPath != "" {
			return po, fmt.Errorf(
				"unexported fields not allowed in dig.In, did you mean to export %q (%v) from %v?",
				f.Name, f.Type, t)
		}

		p, err := newParam(f.Type)
		if err != nil {
			return po, errWrapf(err, "bad field %q of %v", f.Name, t)
		}

		name := f.Tag.Get(_nameTag)
		optional, err := isFieldOptional(t, f)
		if err != nil {
			return po, err
		}

		if sp, ok := p.(paramSingle); ok {
			// Field tags apply only if the field is "simple"
			sp.Name = name
			sp.Optional = optional
			p = sp
		}

		po.Fields = append(po.Fields, paramObjectField{
			FieldName:  f.Name,
			FieldIndex: i,
			Param:      p,
		})
		po.deps = append(po.deps, p.Dependencies()...)
	}
	return po, nil
}

func (po paramObject) Dependencies() []edge { return po.deps }
<commit_msg>Remove param.Dependencies() (#154)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package dig

import (
	"fmt"
	"reflect"
)

// The param interface represents a dependency for a constructor.
//
// The following implementations exist:
//  paramList     All arguments of the constructor.
//  paramSingle   An explicitly requested type.
//  paramObject   dig.In struct where each field in the struct can be another
//                param.
type param interface {
	fmt.Stringer
}

var (
	_ param = paramSingle{}
	_ param = paramObject{}
	_ param = paramList{}
)

// Calls the provided function on all paramSingles in the given param tree.
func forEachParamSingle(param param, f func(paramSingle)) {
	switch p := param.(type) {
	case paramList:
		for _, arg := range p.Params {
			forEachParamSingle(arg, f)
		}
	case paramSingle:
		f(p)
	case paramObject:
		for _, field := range p.Fields {
			forEachParamSingle(field.Param, f)
		}
	default:
		panic(fmt.Sprintf(
			"It looks like you have found a bug in dig. "+
				"Please file an issue at https://github.com/uber-go/dig/issues/ "+
				"and provide the following message: "+
				"received unknown param type %T", param))
	}
}

// newParam builds a param from the given type. If the provided type is a
// dig.In struct, an paramObject will be returned.
func newParam(t reflect.Type) (param, error) {
	if IsIn(t) {
		return newParamObject(t)
	}
	return paramSingle{Type: t}, nil
}

// paramList holds all arguments of the constructor as params.
type paramList struct {
	ctype reflect.Type // type of the constructor

	Params []param
}

// newParamList builds a paramList from the provided constructor type.
//
// Variadic arguments of a constructor are ignored and not included as
// dependencies.
func newParamList(ctype reflect.Type) (paramList, error) {
	numArgs := ctype.NumIn()
	if ctype.IsVariadic() {
		// NOTE: If the function is variadic, we skip the last argument
		// because we're not filling variadic arguments yet. See #120.
		numArgs--
	}

	pl := paramList{
		ctype:  ctype,
		Params: make([]param, 0, numArgs),
	}

	for i := 0; i < numArgs; i++ {
		p, err := newParam(ctype.In(i))
		if err != nil {
			return pl, errWrapf(err, "bad argument %d", i+1)
		}
		pl.Params = append(pl.Params, p)
	}
	return pl, nil
}

// paramSingle is an explicitly requested type, optionally with a name.
//
// This object must be present in the graph as-is unless it's specified as
// optional.
type paramSingle struct {
	Name     string
	Optional bool
	Type     reflect.Type
}

// paramObjectField is a single field of a dig.In struct.
type paramObjectField struct {
	// Name of the field in the struct.
	FieldName string

	// Index of this field in the target struct.
	//
	// We need to track this separately because not all fields of the
	// struct map to params.
	FieldIndex int

	// The dependency requested by this field.
	Param param
}

// paramObject is a dig.In struct where each field is another param.
//
// This object is not expected in the graph as-is.
type paramObject struct {
	Type   reflect.Type
	Fields []paramObjectField

	deps []edge
}

// newParamObject builds an paramObject from the provided type. The type MUST
// be a dig.In struct.
func newParamObject(t reflect.Type) (paramObject, error) {
	po := paramObject{Type: t}

	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		if f.Type == _inType {
			// Skip over the dig.In embed.
			continue
		}

		if f.PkgPath != "" {
			return po, fmt.Errorf(
				"unexported fields not allowed in dig.In, did you mean to export %q (%v) from %v?",
				f.Name, f.Type, t)
		}

		p, err := newParam(f.Type)
		if err != nil {
			return po, errWrapf(err, "bad field %q of %v", f.Name, t)
		}

		name := f.Tag.Get(_nameTag)
		optional, err := isFieldOptional(t, f)
		if err != nil {
			return po, err
		}

		if sp, ok := p.(paramSingle); ok {
			// Field tags apply only if the field is "simple"
			sp.Name = name
			sp.Optional = optional
			p = sp
		}

		po.Fields = append(po.Fields, paramObjectField{
			FieldName:  f.Name,
			FieldIndex: i,
			Param:      p,
		})
	}
	return po, nil
}
<|endoftext|>


<commit_before>package graph

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/engine"
	"github.com/docker/docker/pkg/tarsum"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/runconfig"
	"github.com/docker/libtrust"
)

func (s *TagStore) CmdManifest(job *engine.Job) engine.Status {
	if len(job.Args) != 1 {
		return job.Errorf("usage: %s NAME", job.Name)
	}
	name := job.Args[0]
	tag := job.Getenv("tag")
	if tag == "" {
		tag = "latest"
	}

	// Resolve the Repository name from fqn to endpoint + name
	repoInfo, err := registry.ParseRepositoryInfo(name)
	if err != nil {
		return job.Error(err)
	}

	manifestBytes, err := s.newManifest(name, repoInfo.RemoteName, tag)
	if err != nil {
		return job.Error(err)
	}

	_, err = job.Stdout.Write(manifestBytes)
	if err != nil {
		return job.Error(err)
	}

	return engine.StatusOK
}

func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) {
	manifest := &registry.ManifestData{
		Name:          remoteName,
		Tag:           tag,
		SchemaVersion: 1,
	}
	localRepo, err := s.Get(localName)
	if err != nil {
		return nil, err
	}
	if localRepo == nil {
		return nil, fmt.Errorf("Repo does not exist: %s", localName)
	}

	// Get the top-most layer id which the tag points to
	layerId, exists := localRepo[tag]
	if !exists {
		return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag)
	}
	layersSeen := make(map[string]bool)

	layer, err := s.graph.Get(layerId)
	if err != nil {
		return nil, err
	}
	manifest.Architecture = layer.Architecture
	manifest.FSLayers = make([]*registry.FSLayer, 0, 4)
	manifest.History = make([]*registry.ManifestHistory, 0, 4)
	var metadata runconfig.Config
	if layer.Config != nil {
		metadata = *layer.Config
	}

	for ; layer != nil; layer, err = layer.GetParent() {
		if err != nil {
			return nil, err
		}

		if layersSeen[layer.ID] {
			break
		}
		if layer.Config != nil && metadata.Image != layer.ID {
			err = runconfig.Merge(&metadata, layer.Config)
			if err != nil {
				return nil, err
			}
		}

		archive, err := layer.TarLayer()
		if err != nil {
			return nil, err
		}

		tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1)
		if err != nil {
			return nil, err
		}
		if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
			return nil, err
		}

		tarId := tarSum.Sum(nil)

		manifest.FSLayers = append(manifest.FSLayers, &registry.FSLayer{BlobSum: tarId})

		layersSeen[layer.ID] = true
		jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json"))
		if err != nil {
			return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)
		}
		manifest.History = append(manifest.History, &registry.ManifestHistory{V1Compatibility: string(jsonData)})
	}

	manifestBytes, err := json.MarshalIndent(manifest, "", "   ")
	if err != nil {
		return nil, err
	}

	return manifestBytes, nil
}

func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) {
	sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures")
	if err != nil {
		return nil, false, fmt.Errorf("error parsing payload: %s", err)
	}

	keys, err := sig.Verify()
	if err != nil {
		return nil, false, fmt.Errorf("error verifying payload: %s", err)
	}

	payload, err := sig.Payload()
	if err != nil {
		return nil, false, fmt.Errorf("error retrieving payload: %s", err)
	}

	var manifest registry.ManifestData
	if err := json.Unmarshal(payload, &manifest); err != nil {
		return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
	}
	if manifest.SchemaVersion != 1 {
		return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion)
	}

	var verified bool
	for _, key := range keys {
		job := eng.Job("trust_key_check")
		b, err := key.MarshalJSON()
		if err != nil {
			return nil, false, fmt.Errorf("error marshalling public key: %s", err)
		}
		namespace := manifest.Name
		if namespace[0] != '/' {
			namespace = "/" + namespace
		}
		stdoutBuffer := bytes.NewBuffer(nil)

		job.Args = append(job.Args, namespace)
		job.Setenv("PublicKey", string(b))
		// Check key has read/write permission (0x03)
		job.SetenvInt("Permission", 0x03)
		job.Stdout.Add(stdoutBuffer)
		if err = job.Run(); err != nil {
			return nil, false, fmt.Errorf("error running key check: %s", err)
		}
		result := engine.Tail(stdoutBuffer, 1)
		log.Debugf("Key check result: %q", result)
		if result == "verified" {
			verified = true
		}
	}

	return &manifest, verified, nil
}

func checkValidManifest(manifest *registry.ManifestData) error {
	if len(manifest.FSLayers) != len(manifest.History) {
		return fmt.Errorf("length of history not equal to number of layers")
	}

	if len(manifest.FSLayers) == 0 {
		return fmt.Errorf("no FSLayers in manifest")
	}

	return nil
}
<commit_msg>Use layer checksum if calculated during manifest creation<commit_after>package graph

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/engine"
	"github.com/docker/docker/pkg/tarsum"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/runconfig"
	"github.com/docker/libtrust"
)

func (s *TagStore) CmdManifest(job *engine.Job) engine.Status {
	if len(job.Args) != 1 {
		return job.Errorf("usage: %s NAME", job.Name)
	}
	name := job.Args[0]
	tag := job.Getenv("tag")
	if tag == "" {
		tag = "latest"
	}

	// Resolve the Repository name from fqn to endpoint + name
	repoInfo, err := registry.ParseRepositoryInfo(name)
	if err != nil {
		return job.Error(err)
	}

	manifestBytes, err := s.newManifest(name, repoInfo.RemoteName, tag)
	if err != nil {
		return job.Error(err)
	}

	_, err = job.Stdout.Write(manifestBytes)
	if err != nil {
		return job.Error(err)
	}

	return engine.StatusOK
}

func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) {
	manifest := &registry.ManifestData{
		Name:          remoteName,
		Tag:           tag,
		SchemaVersion: 1,
	}
	localRepo, err := s.Get(localName)
	if err != nil {
		return nil, err
	}
	if localRepo == nil {
		return nil, fmt.Errorf("Repo does not exist: %s", localName)
	}

	// Get the top-most layer id which the tag points to
	layerId, exists := localRepo[tag]
	if !exists {
		return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag)
	}
	layersSeen := make(map[string]bool)

	layer, err := s.graph.Get(layerId)
	if err != nil {
		return nil, err
	}
	manifest.Architecture = layer.Architecture
	manifest.FSLayers = make([]*registry.FSLayer, 0, 4)
	manifest.History = make([]*registry.ManifestHistory, 0, 4)
	var metadata runconfig.Config
	if layer.Config != nil {
		metadata = *layer.Config
	}

	for ; layer != nil; layer, err = layer.GetParent() {
		if err != nil {
			return nil, err
		}

		if layersSeen[layer.ID] {
			break
		}
		if layer.Config != nil && metadata.Image != layer.ID {
			err = runconfig.Merge(&metadata, layer.Config)
			if err != nil {
				return nil, err
			}
		}

		checksum := layer.Checksum
		if tarsum.VersionLabelForChecksum(checksum) != tarsum.Version1.String() {
			archive, err := layer.TarLayer()
			if err != nil {
				return nil, err
			}

			tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1)
			if err != nil {
				return nil, err
			}
			if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
				return nil, err
			}

			checksum = tarSum.Sum(nil)
		}

		jsonData, err := layer.RawJson()
		if err != nil {
			return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)
		}

		manifest.FSLayers = append(manifest.FSLayers, &registry.FSLayer{BlobSum: checksum})

		layersSeen[layer.ID] = true

		manifest.History = append(manifest.History, &registry.ManifestHistory{V1Compatibility: string(jsonData)})
	}

	manifestBytes, err := json.MarshalIndent(manifest, "", "   ")
	if err != nil {
		return nil, err
	}

	return manifestBytes, nil
}

func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) {
	sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures")
	if err != nil {
		return nil, false, fmt.Errorf("error parsing payload: %s", err)
	}

	keys, err := sig.Verify()
	if err != nil {
		return nil, false, fmt.Errorf("error verifying payload: %s", err)
	}

	payload, err := sig.Payload()
	if err != nil {
		return nil, false, fmt.Errorf("error retrieving payload: %s", err)
	}

	var manifest registry.ManifestData
	if err := json.Unmarshal(payload, &manifest); err != nil {
		return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
	}
	if manifest.SchemaVersion != 1 {
		return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion)
	}

	var verified bool
	for _, key := range keys {
		job := eng.Job("trust_key_check")
		b, err := key.MarshalJSON()
		if err != nil {
			return nil, false, fmt.Errorf("error marshalling public key: %s", err)
		}
		namespace := manifest.Name
		if namespace[0] != '/' {
			namespace = "/" + namespace
		}
		stdoutBuffer := bytes.NewBuffer(nil)

		job.Args = append(job.Args, namespace)
		job.Setenv("PublicKey", string(b))
		// Check key has read/write permission (0x03)
		job.SetenvInt("Permission", 0x03)
		job.Stdout.Add(stdoutBuffer)
		if err = job.Run(); err != nil {
			return nil, false, fmt.Errorf("error running key check: %s", err)
		}
		result := engine.Tail(stdoutBuffer, 1)
		log.Debugf("Key check result: %q", result)
		if result == "verified" {
			verified = true
		}
	}

	return &manifest, verified, nil
}

func checkValidManifest(manifest *registry.ManifestData) error {
	if len(manifest.FSLayers) != len(manifest.History) {
		return fmt.Errorf("length of history not equal to number of layers")
	}

	if len(manifest.FSLayers) == 0 {
		return fmt.Errorf("no FSLayers in manifest")
	}

	return nil
}
<|endoftext|>