<commit_before>// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
	"github.com/ipfs/go-datastore/query"

	logging "github.com/ipfs/go-datastore/Godeps/_workspace/src/github.com/ipfs/go-log"
)

var log = logging.Logger("flatfs")

const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
	// length of the dir splay prefix, in bytes of hex digits
	hexPrefixLen int

	// sychronize all writes and directory changes for added safety
	sync bool
}

var _ datastore.Datastore = (*Datastore)(nil)

func New(path string, prefixLen int, sync bool) (*Datastore, error) {
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
		path: path,
		// convert from binary bytes to bytes of hex encoding
		hexPrefixLen: prefixLen * hex.EncodedLen(1),
		sync:         sync,
	}
	return fs, nil
}

var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
	safe := hex.EncodeToString(key.Bytes()[1:])
	prefix := (safe + padding)[:fs.hexPrefixLen]
	dir = path.Join(fs.path, prefix)
	file = path.Join(dir, safe+extension)
	return dir, file
}

func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
	k, err := hex.DecodeString(name)
	if err != nil {
		return datastore.Key{}, false
	}
	return datastore.NewKey(string(k)), true
}

func (fs *Datastore) makePrefixDir(dir string) error {
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

var putMaxRetries = 3

func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
			return nil
		}

		if !strings.Contains(err.Error(), "too many open files") {
			return err
		}

		log.Error("too many open files, retrying in %dms", 100*i)
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
	dir, path := fs.encode(key)
	if err := fs.makePrefixDir(dir); err != nil {
		return err
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

	err = osrename.Rename(tmp.Name(), path)
	if err != nil {
		return err
	}
	removed = true

	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
		}

		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}

	return nil
}

func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
}

func (fs *Datastore) Delete(key datastore.Key) error {
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

	reschan := make(chan query.Result)
	go func() {
		defer close(reschan)
		err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {

			if !info.Mode().IsRegular() || info.Name()[0] == '.' {
				return nil
			}

			key, ok := fs.decode(info.Name())
			if !ok {
				log.Warning("failed to decode entry in flatfs")
				return nil
			}

			reschan <- query.Result{
				Entry: query.Entry{
					Key: key.String(),
				},
			}
			return nil
		})
		if err != nil {
			log.Warning("walk failed: ", err)
		}
	}()
	return query.ResultsWithChan(q, reschan), nil
}

func (fs *Datastore) Close() error {
	return nil
}

type flatfsBatch struct {
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

func (fs *Datastore) Batch() (datastore.Batch, error) {
	return &flatfsBatch{
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
	}, nil
}

func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
	bt.puts[key] = val
	return nil
}

func (bt *flatfsBatch) Delete(key datastore.Key) error {
	bt.deletes[key] = struct{}{}
	return nil
}

func (bt *flatfsBatch) Commit() error {
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}
<commit_msg>fix panic on zero length filename<commit_after>// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
	"github.com/ipfs/go-datastore/query"

	logging "github.com/ipfs/go-datastore/Godeps/_workspace/src/github.com/ipfs/go-log"
)

var log = logging.Logger("flatfs")

const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
	// length of the dir splay prefix, in bytes of hex digits
	hexPrefixLen int

	// sychronize all writes and directory changes for added safety
	sync bool
}

var _ datastore.Datastore = (*Datastore)(nil)

func New(path string, prefixLen int, sync bool) (*Datastore, error) {
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
		path: path,
		// convert from binary bytes to bytes of hex encoding
		hexPrefixLen: prefixLen * hex.EncodedLen(1),
		sync:         sync,
	}
	return fs, nil
}

var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
	safe := hex.EncodeToString(key.Bytes()[1:])
	prefix := (safe + padding)[:fs.hexPrefixLen]
	dir = path.Join(fs.path, prefix)
	file = path.Join(dir, safe+extension)
	return dir, file
}

func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
	k, err := hex.DecodeString(name)
	if err != nil {
		return datastore.Key{}, false
	}
	return datastore.NewKey(string(k)), true
}

func (fs *Datastore) makePrefixDir(dir string) error {
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

var putMaxRetries = 3

func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
			return nil
		}

		if !strings.Contains(err.Error(), "too many open files") {
			return err
		}

		log.Error("too many open files, retrying in %dms", 100*i)
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
	dir, path := fs.encode(key)
	if err := fs.makePrefixDir(dir); err != nil {
		return err
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

	err = osrename.Rename(tmp.Name(), path)
	if err != nil {
		return err
	}
	removed = true

	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
		}

		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}

	return nil
}

func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
}

func (fs *Datastore) Delete(key datastore.Key) error {
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

	reschan := make(chan query.Result)
	go func() {
		defer close(reschan)
		err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {

			if !info.Mode().IsRegular() || strings.HasPrefix(info.Name(), ".") {
				return nil
			}

			key, ok := fs.decode(info.Name())
			if !ok {
				log.Warning("failed to decode entry in flatfs")
				return nil
			}

			reschan <- query.Result{
				Entry: query.Entry{
					Key: key.String(),
				},
			}
			return nil
		})
		if err != nil {
			log.Warning("walk failed: ", err)
		}
	}()
	return query.ResultsWithChan(q, reschan), nil
}

func (fs *Datastore) Close() error {
	return nil
}

type flatfsBatch struct {
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

func (fs *Datastore) Batch() (datastore.Batch, error) {
	return &flatfsBatch{
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
	}, nil
}

func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
	bt.puts[key] = val
	return nil
}

func (bt *flatfsBatch) Delete(key datastore.Key) error {
	bt.deletes[key] = struct{}{}
	return nil
}

func (bt *flatfsBatch) Commit() error {
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}
<|endoftext|>


<commit_before>package main

type Car struct {
    Certificate    *Certificate      `json:"certificate"`         // vehicle certificate issued by the DOT
    CreatedTs      int64             `json:"created_ts"`          // birth date
    Vin            string            `json:"vin"`                 // vehicle identification number
}

type User struct {
    Name           string            `json:"name"`
    KeyringEntries []KeyringEntry    `json:"keyring_entries"`
    Certificate    Certificate       `json:"certificate"`
}

type KeyringEntry struct {
    PublicKey      [32]byte          `json:"public_key"`          // key to lock car
    PrivateKey     [32]byte          `json:"private_key"`         // key to unlock car
    CarTs          int64             `json:"car_ts"`              // car ts to find actual car
}

/*
 * Fahrzeugausweis
 *
 * The car certificate information is attested by the DOT
 */
type Certificate struct {
    Username        string `json:"username"`        // car owners name
    Insurer         string `json:"insurer"`         // the name of an insurance company
    Numberplate     string `json:"numberplate"`     // number plate ('AG 104 739')
    Vin             string `json:"vin"`             // vehicle identification number ('WVW ZZZ 6RZ HY26 0780')
    Color           string `json:"color"`
    Type            string `json:"type"`            // type: 'passenger car', 'truck', ...
    Brand           string `json:"brand"`
}

/*
 * Pruefungsbericht
 * (Form. 13.20 A)
 */
type CarAudit struct {
    CarTs                 int64      `json:"car_ts"`
    NumberOfDoors         string     `json:"number_of_doors"`     // '4+1' for a passenger car
    NumberOfCylinders     int        `json:"number_of_cylinders"` // 3, 4, 6, 8 ?
    NumberOfAxis          int        `json:"number_of_axis"`      // typically 2
    MaxSpeed              int        `json:"max_speed"`           // maximum speed as tested
}<commit_msg>remove cert from user<commit_after>package main

type Car struct {
    Certificate    *Certificate      `json:"certificate"`         // vehicle certificate issued by the DOT
    CreatedTs      int64             `json:"created_ts"`          // birth date
    Vin            string            `json:"vin"`                 // vehicle identification number
}

type User struct {
    Name           string            `json:"name"`
    KeyringEntries []KeyringEntry    `json:"keyring_entries"`
}

type KeyringEntry struct {
    PublicKey      [32]byte          `json:"public_key"`          // key to lock car
    PrivateKey     [32]byte          `json:"private_key"`         // key to unlock car
    CarTs          int64             `json:"car_ts"`              // car ts to find actual car
}

/*
 * Fahrzeugausweis
 *
 * The car certificate information is attested by the DOT
 */
type Certificate struct {
    Username        string `json:"username"`        // car owners name
    Insurer         string `json:"insurer"`         // the name of an insurance company
    Numberplate     string `json:"numberplate"`     // number plate ('AG 104 739')
    Vin             string `json:"vin"`             // vehicle identification number ('WVW ZZZ 6RZ HY26 0780')
    Color           string `json:"color"`
    Type            string `json:"type"`            // type: 'passenger car', 'truck', ...
    Brand           string `json:"brand"`
}

/*
 * Pruefungsbericht
 * (Form. 13.20 A)
 */
type CarAudit struct {
    CarTs                 int64      `json:"car_ts"`
    NumberOfDoors         string     `json:"number_of_doors"`     // '4+1' for a passenger car
    NumberOfCylinders     int        `json:"number_of_cylinders"` // 3, 4, 6, 8 ?
    NumberOfAxis          int        `json:"number_of_axis"`      // typically 2
    MaxSpeed              int        `json:"max_speed"`           // maximum speed as tested
}<|endoftext|>


<commit_before>package lclip

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"

	"github.com/mitchellh/go-homedir"
)

var tempPath string

func TestMain(m *testing.M) {
	f, err := ioutil.TempFile(os.TempDir(), "test")
	if err != nil {
		fmt.Fprintln(os.Stderr, "lclip_test:", err)
		return
	}
	f.Close()
	tempPath = f.Name()

	e := m.Run()
	defer os.Exit(e)

	os.Remove(tempPath)
}

func TestDefaultPath(t *testing.T) {
	h, err := homedir.Dir()
	if err != nil {
		t.Fatal(err)
	}
	expect := filepath.Join(h, ".lclip.json")
	actual, err := DefaultPath()
	if err != nil {
		t.Errorf("DefaultPath returns %q; want nil", err)
	}
	if actual != expect {
		t.Errorf("DefaultPath = %q; want %q",
			actual, expect)
	}
}

func TestCreateStorageFileIfNotExists(t *testing.T) {
	f, err := ioutil.TempFile(os.TempDir(), "test")
	if err != nil {
		t.Fatal(err)
	}
	if err = f.Close(); err != nil {
		t.Fatal(err)
	}

	path := f.Name()
	if err = os.Remove(path); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(path)

	c, err := NewClipboard(path)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	defer c.Close()

	expect := []byte("{}\n")
	actual, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q; want %q",
			actual, expect)
	}
}

type GetTextTest struct {
	Src string
	Dst []byte
}

var indexTestsGetText = []GetTextTest{
	{Src: "foo", Dst: []byte("bar")},
	{Src: "hoge", Dst: []byte("piyo")},
}

func TestGetText(t *testing.T) {
	w := bytes.NewBuffer(make([]byte, 0))
	m := make(map[string][]byte)
	for _, test := range indexTestsGetText {
		m[test.Src] = test.Dst
	}
	if err := json.NewEncoder(w).Encode(m); err != nil {
		t.Fatal(err)
	}
	if err := ioutil.WriteFile(tempPath, w.Bytes(), 0644); err != nil {
		t.Fatal(err)
	}

	c, err := NewClipboard(tempPath)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	defer c.Close()
	for _, test := range indexTestsGetText {
		expect := test.Dst
		actual := c.Get(test.Src)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Get(%q) = %q; want %q",
				test.Src, actual, expect)
		}
	}
}

type SetTextTest struct {
	Label string
	Data  []byte
}

var indexTestsSetText = []SetTextTest{
	{Label: "a", Data: []byte("aaa")},
	{Label: "abc", Data: []byte("def")},
	{Label: "", Data: []byte("")},
}

func TestSetText(t *testing.T) {
	if err := ioutil.WriteFile(tempPath, []byte(`{}`), 0644); err != nil {
		t.Fatal(err)
	}

	c, err := NewClipboard(tempPath)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	defer c.Close()
	for _, test := range indexTestsSetText {
		c.Set(test.Label, test.Data)
		expect := test.Data
		actual := c.Get(test.Label)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("after Set(%q, %q), Get(%q) = %q; want %q",
				test.Label, test.Data,
				test.Label, actual, expect)
		}
	}
}

var indexTestsLabels = [][]string{
	{"foo", "bar", "baz"},
	{"hoge", "piyo", "fuga"},
}

func TestListLabels(t *testing.T) {
	empty := []byte(`{}`)
	for _, labels := range indexTestsLabels {
		if err := ioutil.WriteFile(tempPath, empty, 0644); err != nil {
			t.Fatal(err)
		}

		c, err := NewClipboard(tempPath)
		if err != nil {
			t.Errorf("NewClipboard returns %q; want nil", err)
		}
		for _, label := range labels {
			c.Set(label, []byte(``))
		}

		expect := append(make([]string, 0, len(labels)), labels...)
		actual := c.Labels()
		sort.Strings(expect)
		sort.Strings(actual)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("got %q; want %q", actual, expect)
		}
		if err := c.Close(); err != nil {
			t.Errorf("Close returns %q; want nil", err)
		}
	}
}

func TestSaveText(t *testing.T) {
	if err := ioutil.WriteFile(tempPath, []byte(`{}`), 0644); err != nil {
		t.Fatal(err)
	}

	k, v := "key", []byte("value")
	{
		c, err := NewClipboard(tempPath)
		if err != nil {
			t.Errorf("NewClipboard returns %q; want nil", err)
		}
		c.Set(k, v)
		if err := c.Close(); err != nil {
			t.Errorf("Close returns %q; want nil", err)
		}
	}
	{
		c, err := NewClipboard(tempPath)
		if err != nil {
			t.Errorf("NewClipboard returns %q; want nil", err)
		}
		defer c.Close()
		expect := v
		actual := c.Get(k)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Get(%q) = %q; want %q",
				k, actual, expect)
		}
	}
}
<commit_msg>Add test for change storage path<commit_after>package lclip

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"

	"github.com/mitchellh/go-homedir"
)

var tempPath string

func TestMain(m *testing.M) {
	f, err := ioutil.TempFile(os.TempDir(), "test")
	if err != nil {
		fmt.Fprintln(os.Stderr, "lclip_test:", err)
		return
	}
	f.Close()
	tempPath = f.Name()

	e := m.Run()
	defer os.Exit(e)

	os.Remove(tempPath)
}

func TestDefaultPath(t *testing.T) {
	h, err := homedir.Dir()
	if err != nil {
		t.Fatal(err)
	}
	expect := filepath.Join(h, ".lclip.db")
	actual, err := DefaultPath()
	if err != nil {
		t.Errorf("DefaultPath returns %q; want nil", err)
	}
	if actual != expect {
		t.Errorf("DefaultPath = %q; want %q",
			actual, expect)
	}
}

func TestCreateStorageFileIfNotExists(t *testing.T) {
	f, err := ioutil.TempFile(os.TempDir(), "test")
	if err != nil {
		t.Fatal(err)
	}
	if err = f.Close(); err != nil {
		t.Fatal(err)
	}

	path := f.Name()
	if err = os.Remove(path); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(path)

	c, err := NewClipboard(path)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	defer c.Close()

	expect := []byte("{}\n")
	actual, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got %q; want %q",
			actual, expect)
	}
}

type GetTextTest struct {
	Src string
	Dst []byte
}

var indexTestsGetText = []GetTextTest{
	{Src: "foo", Dst: []byte("bar")},
	{Src: "hoge", Dst: []byte("piyo")},
}

func TestGetText(t *testing.T) {
	w := bytes.NewBuffer(make([]byte, 0))
	m := make(map[string][]byte)
	for _, test := range indexTestsGetText {
		m[test.Src] = test.Dst
	}
	if err := json.NewEncoder(w).Encode(m); err != nil {
		t.Fatal(err)
	}
	if err := ioutil.WriteFile(tempPath, w.Bytes(), 0644); err != nil {
		t.Fatal(err)
	}

	c, err := NewClipboard(tempPath)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	defer c.Close()
	for _, test := range indexTestsGetText {
		expect := test.Dst
		actual := c.Get(test.Src)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Get(%q) = %q; want %q",
				test.Src, actual, expect)
		}
	}
}

type SetTextTest struct {
	Label string
	Data  []byte
}

var indexTestsSetText = []SetTextTest{
	{Label: "a", Data: []byte("aaa")},
	{Label: "abc", Data: []byte("def")},
	{Label: "", Data: []byte("")},
}

func TestSetText(t *testing.T) {
	if err := ioutil.WriteFile(tempPath, []byte(`{}`), 0644); err != nil {
		t.Fatal(err)
	}

	c, err := NewClipboard(tempPath)
	if err != nil {
		t.Errorf("NewClipboard returns %q; want nil", err)
	}
	defer c.Close()
	for _, test := range indexTestsSetText {
		c.Set(test.Label, test.Data)
		expect := test.Data
		actual := c.Get(test.Label)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("after Set(%q, %q), Get(%q) = %q; want %q",
				test.Label, test.Data,
				test.Label, actual, expect)
		}
	}
}

var indexTestsLabels = [][]string{
	{"foo", "bar", "baz"},
	{"hoge", "piyo", "fuga"},
}

func TestListLabels(t *testing.T) {
	empty := []byte(`{}`)
	for _, labels := range indexTestsLabels {
		if err := ioutil.WriteFile(tempPath, empty, 0644); err != nil {
			t.Fatal(err)
		}

		c, err := NewClipboard(tempPath)
		if err != nil {
			t.Errorf("NewClipboard returns %q; want nil", err)
		}
		for _, label := range labels {
			c.Set(label, []byte(``))
		}

		expect := append(make([]string, 0, len(labels)), labels...)
		actual := c.Labels()
		sort.Strings(expect)
		sort.Strings(actual)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("got %q; want %q", actual, expect)
		}
		if err := c.Close(); err != nil {
			t.Errorf("Close returns %q; want nil", err)
		}
	}
}

func TestSaveText(t *testing.T) {
	if err := ioutil.WriteFile(tempPath, []byte(`{}`), 0644); err != nil {
		t.Fatal(err)
	}

	k, v := "key", []byte("value")
	{
		c, err := NewClipboard(tempPath)
		if err != nil {
			t.Errorf("NewClipboard returns %q; want nil", err)
		}
		c.Set(k, v)
		if err := c.Close(); err != nil {
			t.Errorf("Close returns %q; want nil", err)
		}
	}
	{
		c, err := NewClipboard(tempPath)
		if err != nil {
			t.Errorf("NewClipboard returns %q; want nil", err)
		}
		defer c.Close()
		expect := v
		actual := c.Get(k)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Get(%q) = %q; want %q",
				k, actual, expect)
		}
	}
}
<|endoftext|>


<commit_before>package form

import (
	"math"
	"reflect"
	"regexp"
	"strconv"
)

type processor interface {
	process(reflect.Value, []string) error
}

type inum struct {
	min, max int64
	bits     int
}

func newInum(tags reflect.StructTag, bits int) inum {
	i := inum{
		min:  math.MinInt64,
		max:  math.MaxInt64,
		bits: bits,
	}
	if m := tags.Get("min"); m != "" {
		im, err := strconv.ParseInt(m, 10, bits)
		if err == nil {
			i.min = im
		}
	}
	if m := tags.Get("max"); m != "" {
		im, err := strconv.ParseInt(m, 10, bits)
		if err == nil {
			i.max = im
		}
	}
	return i
}

func (i inum) process(v reflect.Value, data []string) error {
	num, err := strconv.ParseInt(data[0], 10, i.bits)
	if err != nil {
		return err
	}
	if num < i.min || num > i.max {
		return ErrNotInRange
	}
	v.SetInt(num)
	return nil
}

type unum struct {
	min, max uint64
	bits     int
}

func newUnum(tags reflect.StructTag, bits int) unum {
	u := unum{
		max:  math.MaxUint64,
		bits: bits,
	}
	if m := tags.Get("min"); m != "" {
		um, err := strconv.ParseUint(m, 10, bits)
		if err == nil {
			u.min = um
		}
	}
	if m := tags.Get("max"); m != "" {
		um, err := strconv.ParseUint(m, 10, bits)
		if err == nil {
			u.max = um
		}
	}
	return u
}

func (u unum) process(v reflect.Value, data []string) error {
	num, err := strconv.ParseUint(data[0], 10, u.bits)
	if err != nil {
		return err
	}
	if num < u.min || num > u.max {
		return ErrNotInRange
	}
	v.SetUint(num)
	return nil
}

type float struct {
	min, max float64
	bits     int
}

func newFloat(tags reflect.StructTag, bits int) float {
	f := float{
		min:  -math.MaxFloat64,
		max:  math.MaxFloat64,
		bits: bits,
	}
	if m := tags.Get("min"); m != "" {
		um, err := strconv.ParseFloat(m, bits)
		if err == nil {
			f.min = um
		}
	}
	if m := tags.Get("max"); m != "" {
		um, err := strconv.ParseFloat(m, bits)
		if err == nil {
			f.max = um
		}
	}
	return f
}

func (f float) process(v reflect.Value, data []string) error {
	num, err := strconv.ParseFloat(data[0], f.bits)
	if err != nil {
		return err
	}
	if num < f.min || num > f.max {
		return ErrNotInRange
	}
	v.SetFloat(num)
	return nil
}

type str struct {
	regex *regexp.Regexp
}

func newString(tags reflect.StructTag) str {
	if r := tags.Get("regex"); r != "" {
		if re, err := regexp.Compile(r); err == nil {
			return str{
				regex: re,
			}
		}
	}
	return str{}
}

func (s str) process(v reflect.Value, data []string) error {
	if s.regex != nil {
		if !s.regex.MatchString(data[0]) {
			return ErrNoMatch
		}
	}
	v.SetString(data[0])
	return nil
}

type boolean struct{}

func matchString(a string, b []byte) bool {
	if len(a) != len(b) {
		return false
	}
	for n, c := range b {
		if a[n] != c && a[n] != c|32 {
			return false
		}
	}
	return true
}

var (
	trues = [...][]byte{
		{'1'},
		{'Y'},
		{'T'},
		{'O', 'N'},
		{'Y', 'E', 'S'},
		{'T', 'R', 'U', 'E'},
	}
	falses = [...][]byte{
		{'0'},
		{'N'},
		{'F'},
		{'O', 'F', 'F'},
		{'N', 'O'},
		{'F', 'A', 'L', 'S', 'E'},
	}
)

func (boolean) process(v reflect.Value, data []string) error {
	for _, b := range trues {
		if matchString(data[0], b) {
			v.SetBool(true)
			return nil
		}
	}
	for _, b := range falses {
		if matchString(data[0], b) {
			v.SetBool(false)
			return nil
		}
	}
	return ErrInvalidBoolean
}

type slice struct {
	processor
	typ reflect.Type
}

func (s slice) process(v reflect.Value, data []string) error {
	v.Set(reflect.MakeSlice(s.typ, 1, len(data)))
	l := 0
	errs := make(Errors, 0)
	for n := range data {
		if err := s.processor.process(v.Index(l), data[n:]); err != nil {
			errs = append(errs, err)
		} else if l < len(data) {
			l++
			v.SetLen(l + 1)
		}
	}
	if len(errs) > 0 {
		return errs
	}
	return nil
}

type pointer struct {
	processor
	typ reflect.Type
}

func (p pointer) process(v reflect.Value, data []string) error {
	pv := reflect.New(p.typ)
	if err := p.processor.process(pv.Elem(), data); err != nil {
		return err
	}
	v.Set(pv)
	return nil
}

type formParser interface {
	ParseForm([]string) error
}

type inter bool

func (i inter) process(v reflect.Value, data []string) error {
	if i {
		v = v.Addr()
	}
	return v.Interface().(formParser).ParseForm(data)
}
<commit_msg>rewrote boolean checking to avoid allocation<commit_after>package form

import (
	"math"
	"reflect"
	"regexp"
	"strconv"
)

type processor interface {
	process(reflect.Value, []string) error
}

type inum struct {
	min, max int64
	bits     int
}

func newInum(tags reflect.StructTag, bits int) inum {
	i := inum{
		min:  math.MinInt64,
		max:  math.MaxInt64,
		bits: bits,
	}
	if m := tags.Get("min"); m != "" {
		im, err := strconv.ParseInt(m, 10, bits)
		if err == nil {
			i.min = im
		}
	}
	if m := tags.Get("max"); m != "" {
		im, err := strconv.ParseInt(m, 10, bits)
		if err == nil {
			i.max = im
		}
	}
	return i
}

func (i inum) process(v reflect.Value, data []string) error {
	num, err := strconv.ParseInt(data[0], 10, i.bits)
	if err != nil {
		return err
	}
	if num < i.min || num > i.max {
		return ErrNotInRange
	}
	v.SetInt(num)
	return nil
}

type unum struct {
	min, max uint64
	bits     int
}

func newUnum(tags reflect.StructTag, bits int) unum {
	u := unum{
		max:  math.MaxUint64,
		bits: bits,
	}
	if m := tags.Get("min"); m != "" {
		um, err := strconv.ParseUint(m, 10, bits)
		if err == nil {
			u.min = um
		}
	}
	if m := tags.Get("max"); m != "" {
		um, err := strconv.ParseUint(m, 10, bits)
		if err == nil {
			u.max = um
		}
	}
	return u
}

func (u unum) process(v reflect.Value, data []string) error {
	num, err := strconv.ParseUint(data[0], 10, u.bits)
	if err != nil {
		return err
	}
	if num < u.min || num > u.max {
		return ErrNotInRange
	}
	v.SetUint(num)
	return nil
}

type float struct {
	min, max float64
	bits     int
}

func newFloat(tags reflect.StructTag, bits int) float {
	f := float{
		min:  -math.MaxFloat64,
		max:  math.MaxFloat64,
		bits: bits,
	}
	if m := tags.Get("min"); m != "" {
		um, err := strconv.ParseFloat(m, bits)
		if err == nil {
			f.min = um
		}
	}
	if m := tags.Get("max"); m != "" {
		um, err := strconv.ParseFloat(m, bits)
		if err == nil {
			f.max = um
		}
	}
	return f
}

func (f float) process(v reflect.Value, data []string) error {
	num, err := strconv.ParseFloat(data[0], f.bits)
	if err != nil {
		return err
	}
	if num < f.min || num > f.max {
		return ErrNotInRange
	}
	v.SetFloat(num)
	return nil
}

type str struct {
	regex *regexp.Regexp
}

func newString(tags reflect.StructTag) str {
	if r := tags.Get("regex"); r != "" {
		if re, err := regexp.Compile(r); err == nil {
			return str{
				regex: re,
			}
		}
	}
	return str{}
}

func (s str) process(v reflect.Value, data []string) error {
	if s.regex != nil {
		if !s.regex.MatchString(data[0]) {
			return ErrNoMatch
		}
	}
	v.SetString(data[0])
	return nil
}

type boolean struct{}

func matchString(a string, b []byte) bool {
	if len(a) != len(b) {
		return false
	}
	for n, c := range b {
		if a[n]|32 != c {
			return false
		}
	}
	return true
}

var (
	trues = [...][]byte{
		{'1'},
		{'y'},
		{'t'},
		{'o', 'n'},
		{'y', 'e', 's'},
		{'t', 'r', 'u', 'e'},
	}
	falses = [...][]byte{
		{'0'},
		{'n'},
		{'g'},
		{'o', 'f', 'f'},
		{'n', 'o'},
		{'f', 'a', 'l', 's', 'e'},
	}
)

func (boolean) process(v reflect.Value, data []string) error {
	for _, b := range trues {
		if matchString(data[0], b) {
			v.SetBool(true)
			return nil
		}
	}
	for _, b := range falses {
		if matchString(data[0], b) {
			v.SetBool(false)
			return nil
		}
	}
	return ErrInvalidBoolean
}

type slice struct {
	processor
	typ reflect.Type
}

func (s slice) process(v reflect.Value, data []string) error {
	v.Set(reflect.MakeSlice(s.typ, 1, len(data)))
	l := 0
	errs := make(Errors, 0)
	for n := range data {
		if err := s.processor.process(v.Index(l), data[n:]); err != nil {
			errs = append(errs, err)
		} else if l < len(data) {
			l++
			v.SetLen(l + 1)
		}
	}
	if len(errs) > 0 {
		return errs
	}
	return nil
}

type pointer struct {
	processor
	typ reflect.Type
}

func (p pointer) process(v reflect.Value, data []string) error {
	pv := reflect.New(p.typ)
	if err := p.processor.process(pv.Elem(), data); err != nil {
		return err
	}
	v.Set(pv)
	return nil
}

type formParser interface {
	ParseForm([]string) error
}

type inter bool

func (i inter) process(v reflect.Value, data []string) error {
	if i {
		v = v.Addr()
	}
	return v.Interface().(formParser).ParseForm(data)
}
<|endoftext|>


<commit_before>package containerimage

import (
	"context"
	"encoding/json"
	"runtime"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/diff"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/moby/buildkit/cache"
	gw "github.com/moby/buildkit/frontend/gateway/client"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/flightcontrol"
	"github.com/moby/buildkit/util/imageutil"
	"github.com/moby/buildkit/util/pull"
	"github.com/moby/buildkit/util/resolver"
	"github.com/moby/buildkit/util/winlayers"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// TODO: break apart containerd specifics like contentstore so the resolver
// code can be used with any implementation

type SourceOpt struct {
	Snapshotter   snapshot.Snapshotter
	ContentStore  content.Store
	Applier       diff.Applier
	CacheAccessor cache.Accessor
	ImageStore    images.Store // optional
	ResolverOpt   resolver.ResolveOptionsFunc
}

type imageSource struct {
	SourceOpt
	g flightcontrol.Group
}

func NewSource(opt SourceOpt) (source.Source, error) {
	is := &imageSource{
		SourceOpt: opt,
	}

	return is, nil
}

func (is *imageSource) ID() string {
	return source.DockerImageScheme
}

func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) {
	type t struct {
		dgst digest.Digest
		dt   []byte
	}
	key := ref
	if platform := opt.Platform; platform != nil {
		key += platforms.Format(*platform)
	}

	rm, err := source.ParseImageResolveMode(opt.ResolveMode)
	if err != nil {
		return "", nil, err
	}

	res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) {
		dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.ResolverOpt, sm, is.ImageStore, rm, ref), is.ContentStore, opt.Platform)
		if err != nil {
			return nil, err
		}
		return &t{dgst: dgst, dt: dt}, nil
	})
	if err != nil {
		return "", nil, err
	}
	typed := res.(*t)
	return typed.dgst, typed.dt, nil
}

func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) {
	imageIdentifier, ok := id.(*source.ImageIdentifier)
	if !ok {
		return nil, errors.Errorf("invalid image identifier %v", id)
	}

	platform := platforms.DefaultSpec()
	if imageIdentifier.Platform != nil {
		platform = *imageIdentifier.Platform
	}

	pullerUtil := &pull.Puller{
		Snapshotter:  is.Snapshotter,
		ContentStore: is.ContentStore,
		Applier:      is.Applier,
		Src:          imageIdentifier.Reference,
		Resolver:     pull.NewResolver(ctx, is.ResolverOpt, sm, is.ImageStore, imageIdentifier.ResolveMode, imageIdentifier.Reference.String()),
		Platform:     &platform,
	}
	p := &puller{
		CacheAccessor: is.CacheAccessor,
		Puller:        pullerUtil,
		Platform:      platform,
		id:            imageIdentifier,
	}
	return p, nil
}

type puller struct {
	CacheAccessor cache.Accessor
	Platform      specs.Platform
	id            *source.ImageIdentifier
	*pull.Puller
}

func mainManifestKey(ctx context.Context, desc specs.Descriptor, platform specs.Platform) (digest.Digest, error) {
	dt, err := json.Marshal(struct {
		Digest  digest.Digest
		OS      string
		Arch    string
		Variant string `json:",omitempty"`
	}{
		Digest:  desc.Digest,
		OS:      platform.OS,
		Arch:    platform.Architecture,
		Variant: platform.Variant,
	})
	if err != nil {
		return "", err
	}
	return digest.FromBytes(dt), nil
}

func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error) {
	_, desc, err := p.Puller.Resolve(ctx)
	if err != nil {
		return "", false, err
	}
	if index == 0 || desc.Digest == "" {
		k, err := mainManifestKey(ctx, desc, p.Platform)
		if err != nil {
			return "", false, err
		}
		return k.String(), false, nil
	}
	ref, err := reference.ParseNormalizedNamed(p.Src.String())
	if err != nil {
		return "", false, err
	}
	ref, err = reference.WithDigest(ref, desc.Digest)
	if err != nil {
		return "", false, nil
	}
	_, dt, err := imageutil.Config(ctx, ref.String(), p.Resolver, p.ContentStore, &p.Platform)
	if err != nil {
		// this happens on schema1 images
		k, err := mainManifestKey(ctx, desc, p.Platform)
		if err != nil {
			return "", false, err
		}
		return k.String(), true, nil
	}
	return cacheKeyFromConfig(dt).String(), true, nil
}

func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
	layerNeedsTypeWindows := false
	if platform := p.Puller.Platform; platform != nil {
		if platform.OS == "windows" && runtime.GOOS != "windows" {
			ctx = winlayers.UseWindowsLayerMode(ctx)
			layerNeedsTypeWindows = true
		}
	}

	pulled, err := p.Puller.Pull(ctx)
	if err != nil {
		return nil, err
	}
	if pulled.ChainID == "" {
		return nil, nil
	}
	ref, err := p.CacheAccessor.GetFromSnapshotter(ctx, string(pulled.ChainID), cache.WithDescription("pulled from "+pulled.Ref))
	if err != nil {
		return nil, err
	}

	if layerNeedsTypeWindows && ref != nil {
		if err := markRefLayerTypeWindows(ref); err != nil {
			ref.Release(context.TODO())
			return nil, err
		}
	}

	if p.id.RecordType != "" && cache.GetRecordType(ref) == "" {
		if err := cache.SetRecordType(ref, p.id.RecordType); err != nil {
			ref.Release(context.TODO())
			return nil, err
		}
	}

	return ref, nil
}

func markRefLayerTypeWindows(ref cache.ImmutableRef) error {
	if parent := ref.Parent(); parent != nil {
		defer parent.Release(context.TODO())
		if err := markRefLayerTypeWindows(parent); err != nil {
			return err
		}
	}
	return cache.SetLayerType(ref, "windows")
}

// cacheKeyFromConfig returns a stable digest from image config. If image config
// is a known oci image we will use chainID of layers.
func cacheKeyFromConfig(dt []byte) digest.Digest {
	var img specs.Image
	err := json.Unmarshal(dt, &img)
	if err != nil {
		return digest.FromBytes(dt)
	}
	if img.RootFS.Type != "layers" {
		return digest.FromBytes(dt)
	}
	return identity.ChainID(img.RootFS.DiffIDs)
}
<commit_msg>pull: fix second cachekey for schema1<commit_after>package containerimage

import (
	"context"
	"encoding/json"
	"runtime"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/diff"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/moby/buildkit/cache"
	gw "github.com/moby/buildkit/frontend/gateway/client"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/flightcontrol"
	"github.com/moby/buildkit/util/imageutil"
	"github.com/moby/buildkit/util/pull"
	"github.com/moby/buildkit/util/resolver"
	"github.com/moby/buildkit/util/winlayers"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// TODO: break apart containerd specifics like contentstore so the resolver
// code can be used with any implementation

type SourceOpt struct {
	Snapshotter   snapshot.Snapshotter
	ContentStore  content.Store
	Applier       diff.Applier
	CacheAccessor cache.Accessor
	ImageStore    images.Store // optional
	ResolverOpt   resolver.ResolveOptionsFunc
}

type imageSource struct {
	SourceOpt
	g flightcontrol.Group
}

func NewSource(opt SourceOpt) (source.Source, error) {
	is := &imageSource{
		SourceOpt: opt,
	}

	return is, nil
}

func (is *imageSource) ID() string {
	return source.DockerImageScheme
}

func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) {
	type t struct {
		dgst digest.Digest
		dt   []byte
	}
	key := ref
	if platform := opt.Platform; platform != nil {
		key += platforms.Format(*platform)
	}

	rm, err := source.ParseImageResolveMode(opt.ResolveMode)
	if err != nil {
		return "", nil, err
	}

	res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) {
		dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.ResolverOpt, sm, is.ImageStore, rm, ref), is.ContentStore, opt.Platform)
		if err != nil {
			return nil, err
		}
		return &t{dgst: dgst, dt: dt}, nil
	})
	if err != nil {
		return "", nil, err
	}
	typed := res.(*t)
	return typed.dgst, typed.dt, nil
}

func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) {
	imageIdentifier, ok := id.(*source.ImageIdentifier)
	if !ok {
		return nil, errors.Errorf("invalid image identifier %v", id)
	}

	platform := platforms.DefaultSpec()
	if imageIdentifier.Platform != nil {
		platform = *imageIdentifier.Platform
	}

	pullerUtil := &pull.Puller{
		Snapshotter:  is.Snapshotter,
		ContentStore: is.ContentStore,
		Applier:      is.Applier,
		Src:          imageIdentifier.Reference,
		Resolver:     pull.NewResolver(ctx, is.ResolverOpt, sm, is.ImageStore, imageIdentifier.ResolveMode, imageIdentifier.Reference.String()),
		Platform:     &platform,
	}
	p := &puller{
		CacheAccessor: is.CacheAccessor,
		Puller:        pullerUtil,
		Platform:      platform,
		id:            imageIdentifier,
	}
	return p, nil
}

type puller struct {
	CacheAccessor cache.Accessor
	Platform      specs.Platform
	id            *source.ImageIdentifier
	*pull.Puller
}

func mainManifestKey(ctx context.Context, desc specs.Descriptor, platform specs.Platform) (digest.Digest, error) {
	dt, err := json.Marshal(struct {
		Digest  digest.Digest
		OS      string
		Arch    string
		Variant string `json:",omitempty"`
	}{
		Digest:  desc.Digest,
		OS:      platform.OS,
		Arch:    platform.Architecture,
		Variant: platform.Variant,
	})
	if err != nil {
		return "", err
	}
	return digest.FromBytes(dt), nil
}

func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error) {
	_, desc, err := p.Puller.Resolve(ctx)
	if err != nil {
		return "", false, err
	}
	if index == 0 || desc.Digest == "" {
		k, err := mainManifestKey(ctx, desc, p.Platform)
		if err != nil {
			return "", false, err
		}
		return k.String(), false, nil
	}
	ref, err := reference.ParseNormalizedNamed(p.Src.String())
	if err != nil {
		return "", false, err
	}
	ref, err = reference.WithDigest(ref, desc.Digest)
	if err != nil {
		return "", false, nil
	}
	_, dt, err := imageutil.Config(ctx, ref.String(), p.Resolver, p.ContentStore, &p.Platform)
	if err != nil {
		return "", false, err
	}

	k := cacheKeyFromConfig(dt).String()
	if k == "" {
		k, err := mainManifestKey(ctx, desc, p.Platform)
		if err != nil {
			return "", false, err
		}
		return k.String(), true, nil
	}
	return k, true, nil
}

func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
	layerNeedsTypeWindows := false
	if platform := p.Puller.Platform; platform != nil {
		if platform.OS == "windows" && runtime.GOOS != "windows" {
			ctx = winlayers.UseWindowsLayerMode(ctx)
			layerNeedsTypeWindows = true
		}
	}

	pulled, err := p.Puller.Pull(ctx)
	if err != nil {
		return nil, err
	}
	if pulled.ChainID == "" {
		return nil, nil
	}
	ref, err := p.CacheAccessor.GetFromSnapshotter(ctx, string(pulled.ChainID), cache.WithDescription("pulled from "+pulled.Ref))
	if err != nil {
		return nil, err
	}

	if layerNeedsTypeWindows && ref != nil {
		if err := markRefLayerTypeWindows(ref); err != nil {
			ref.Release(context.TODO())
			return nil, err
		}
	}

	if p.id.RecordType != "" && cache.GetRecordType(ref) == "" {
		if err := cache.SetRecordType(ref, p.id.RecordType); err != nil {
			ref.Release(context.TODO())
			return nil, err
		}
	}

	return ref, nil
}

func markRefLayerTypeWindows(ref cache.ImmutableRef) error {
	if parent := ref.Parent(); parent != nil {
		defer parent.Release(context.TODO())
		if err := markRefLayerTypeWindows(parent); err != nil {
			return err
		}
	}
	return cache.SetLayerType(ref, "windows")
}

// cacheKeyFromConfig returns a stable digest from image config. If image config
// is a known oci image we will use chainID of layers.
func cacheKeyFromConfig(dt []byte) digest.Digest {
	var img specs.Image
	err := json.Unmarshal(dt, &img)
	if err != nil {
		return digest.FromBytes(dt)
	}
	if img.RootFS.Type != "layers" || len(img.RootFS.DiffIDs) == 0 {
		return ""
	}
	return identity.ChainID(img.RootFS.DiffIDs)
}
<|endoftext|>


<commit_before>package c

import (
	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/internal"
)

// C lexer.
var C = internal.Register(MustNewLazyLexer(
	&Config{
		Name:      "C",
		Aliases:   []string{"c"},
		Filenames: []string{"*.c", "*.h", "*.idc"},
		MimeTypes: []string{"text/x-chdr", "text/x-csrc"},
		EnsureNL:  true,
	},
	cRules,
))

func cRules() Rules {
	return Rules{
		"whitespace": {
			{`^#if\s+0`, CommentPreproc, Push("if0")},
			{`^#`, CommentPreproc, Push("macro")},
			{`^(\s*(?:/[*].*?[*]/\s*)?)(#if\s+0)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("if0")},
			{`^(\s*(?:/[*].*?[*]/\s*)?)(#)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("macro")},
			{`\n`, Text, nil},
			{`\s+`, Text, nil},
			{`\\\n`, Text, nil},
			{`//(\n|[\w\W]*?[^\\]\n)`, CommentSingle, nil},
			{`/(\\\n)?[*][\w\W]*?[*](\\\n)?/`, CommentMultiline, nil},
			{`/(\\\n)?[*][\w\W]*`, CommentMultiline, nil},
		},
		"statements": {
			{`(L?)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")},
			{`(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')`, ByGroups(LiteralStringAffix, LiteralStringChar, LiteralStringChar, LiteralStringChar), nil},
			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
			{`0x[0-9a-fA-F]+[LlUu]*`, LiteralNumberHex, nil},
			{`0[0-7]+[LlUu]*`, LiteralNumberOct, nil},
			{`\d+[LlUu]*`, LiteralNumberInteger, nil},
			{`\*/`, Error, nil},
			{`[~!%^&*+=|?:<>/-]`, Operator, nil},
			{`[()\[\],.]`, Punctuation, nil},
			{Words(``, `\b`, `asm`, `auto`, `break`, `case`, `const`, `continue`, `default`, `do`, `else`, `enum`, `extern`, `for`, `goto`, `if`, `register`, `restricted`, `return`, `sizeof`, `static`, `struct`, `switch`, `typedef`, `union`, `volatile`, `while`), Keyword, nil},
			{`(bool|int|long|float|short|double|char|unsigned|signed|void)\b`, KeywordType, nil},
			{Words(``, `\b`, `inline`, `_inline`, `__inline`, `naked`, `restrict`, `thread`, `typename`), KeywordReserved, nil},
			{`(__m(128i|128d|128|64))\b`, KeywordReserved, nil},
			{Words(`__`, `\b`, `asm`, `int8`, `based`, `except`, `int16`, `stdcall`, `cdecl`, `fastcall`, `int32`, `declspec`, `finally`, `int64`, `try`, `leave`, `wchar_t`, `w64`, `unaligned`, `raise`, `noop`, `identifier`, `forceinline`, `assume`), KeywordReserved, nil},
			{`(true|false|NULL)\b`, NameBuiltin, nil},
			{`([a-zA-Z_]\w*)(\s*)(:)(?!:)`, ByGroups(NameLabel, Text, Punctuation), nil},
			{`[a-zA-Z_]\w*`, Name, nil},
		},
		"root": {
			Include("whitespace"),
			{`((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)`, ByGroups(UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), Push("function")},
			{`((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)`, ByGroups(UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), nil},
			Default(Push("statement")),
		},
		"statement": {
			Include("whitespace"),
			Include("statements"),
			{`[{}]`, Punctuation, nil},
			{`;`, Punctuation, Pop(1)},
		},
		"function": {
			Include("whitespace"),
			Include("statements"),
			{`;`, Punctuation, nil},
			{`\{`, Punctuation, Push()},
			{`\}`, Punctuation, Pop(1)},
		},
		"string": {
			{`"`, LiteralString, Pop(1)},
			{`\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})`, LiteralStringEscape, nil},
			{`[^\\"\n]+`, LiteralString, nil},
			{`\\\n`, LiteralString, nil},
			{`\\`, LiteralString, nil},
		},
		"macro": {
			{`(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)`, ByGroups(CommentPreproc, Text, CommentPreprocFile), nil},
			{`[^/\n]+`, CommentPreproc, nil},
			{`/[*](.|\n)*?[*]/`, CommentMultiline, nil},
			{`//.*?\n`, CommentSingle, Pop(1)},
			{`/`, CommentPreproc, nil},
			{`(?<=\\)\n`, CommentPreproc, nil},
			{`\n`, CommentPreproc, Pop(1)},
		},
		"if0": {
			{`^\s*#if.*?(?<!\\)\n`, CommentPreproc, Push()},
			{`^\s*#el(?:se|if).*\n`, CommentPreproc, Pop(1)},
			{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
			{`.*?\n`, Comment, nil},
		},
	}
}
<commit_msg>feat(lexers/c): associate with XBM and XPM<commit_after>package c

import (
	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/internal"
)

// C lexer.
var C = internal.Register(MustNewLazyLexer(
	&Config{
		Name:      "C",
		Aliases:   []string{"c"},
		Filenames: []string{"*.c", "*.h", "*.idc", "*.x[bp]m"},
		MimeTypes: []string{"text/x-chdr", "text/x-csrc", "image/x-xbitmap", "image/x-xpixmap"},
		EnsureNL:  true,
	},
	cRules,
))

func cRules() Rules {
	return Rules{
		"whitespace": {
			{`^#if\s+0`, CommentPreproc, Push("if0")},
			{`^#`, CommentPreproc, Push("macro")},
			{`^(\s*(?:/[*].*?[*]/\s*)?)(#if\s+0)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("if0")},
			{`^(\s*(?:/[*].*?[*]/\s*)?)(#)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("macro")},
			{`\n`, Text, nil},
			{`\s+`, Text, nil},
			{`\\\n`, Text, nil},
			{`//(\n|[\w\W]*?[^\\]\n)`, CommentSingle, nil},
			{`/(\\\n)?[*][\w\W]*?[*](\\\n)?/`, CommentMultiline, nil},
			{`/(\\\n)?[*][\w\W]*`, CommentMultiline, nil},
		},
		"statements": {
			{`(L?)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")},
			{`(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')`, ByGroups(LiteralStringAffix, LiteralStringChar, LiteralStringChar, LiteralStringChar), nil},
			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
			{`0x[0-9a-fA-F]+[LlUu]*`, LiteralNumberHex, nil},
			{`0[0-7]+[LlUu]*`, LiteralNumberOct, nil},
			{`\d+[LlUu]*`, LiteralNumberInteger, nil},
			{`\*/`, Error, nil},
			{`[~!%^&*+=|?:<>/-]`, Operator, nil},
			{`[()\[\],.]`, Punctuation, nil},
			{Words(``, `\b`, `asm`, `auto`, `break`, `case`, `const`, `continue`, `default`, `do`, `else`, `enum`, `extern`, `for`, `goto`, `if`, `register`, `restricted`, `return`, `sizeof`, `static`, `struct`, `switch`, `typedef`, `union`, `volatile`, `while`), Keyword, nil},
			{`(bool|int|long|float|short|double|char|unsigned|signed|void)\b`, KeywordType, nil},
			{Words(``, `\b`, `inline`, `_inline`, `__inline`, `naked`, `restrict`, `thread`, `typename`), KeywordReserved, nil},
			{`(__m(128i|128d|128|64))\b`, KeywordReserved, nil},
			{Words(`__`, `\b`, `asm`, `int8`, `based`, `except`, `int16`, `stdcall`, `cdecl`, `fastcall`, `int32`, `declspec`, `finally`, `int64`, `try`, `leave`, `wchar_t`, `w64`, `unaligned`, `raise`, `noop`, `identifier`, `forceinline`, `assume`), KeywordReserved, nil},
			{`(true|false|NULL)\b`, NameBuiltin, nil},
			{`([a-zA-Z_]\w*)(\s*)(:)(?!:)`, ByGroups(NameLabel, Text, Punctuation), nil},
			{`[a-zA-Z_]\w*`, Name, nil},
		},
		"root": {
			Include("whitespace"),
			{`((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)`, ByGroups(UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), Push("function")},
			{`((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)`, ByGroups(UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), nil},
			Default(Push("statement")),
		},
		"statement": {
			Include("whitespace"),
			Include("statements"),
			{`[{}]`, Punctuation, nil},
			{`;`, Punctuation, Pop(1)},
		},
		"function": {
			Include("whitespace"),
			Include("statements"),
			{`;`, Punctuation, nil},
			{`\{`, Punctuation, Push()},
			{`\}`, Punctuation, Pop(1)},
		},
		"string": {
			{`"`, LiteralString, Pop(1)},
			{`\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})`, LiteralStringEscape, nil},
			{`[^\\"\n]+`, LiteralString, nil},
			{`\\\n`, LiteralString, nil},
			{`\\`, LiteralString, nil},
		},
		"macro": {
			{`(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)`, ByGroups(CommentPreproc, Text, CommentPreprocFile), nil},
			{`[^/\n]+`, CommentPreproc, nil},
			{`/[*](.|\n)*?[*]/`, CommentMultiline, nil},
			{`//.*?\n`, CommentSingle, Pop(1)},
			{`/`, CommentPreproc, nil},
			{`(?<=\\)\n`, CommentPreproc, nil},
			{`\n`, CommentPreproc, Pop(1)},
		},
		"if0": {
			{`^\s*#if.*?(?<!\\)\n`, CommentPreproc, Push()},
			{`^\s*#el(?:se|if).*\n`, CommentPreproc, Pop(1)},
			{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
			{`.*?\n`, Comment, nil},
		},
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/ThomsonReutersEikon/go-ntlm/ntlm"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

var (
	Config        = NewConfig()
	defaultRemote = "origin"
)

// FetchPruneConfig collects together the config options that control fetching and pruning
type FetchPruneConfig struct {
	// The number of days prior to current date for which (local) refs other than HEAD
	// will be fetched with --recent (default 7, 0 = only fetch HEAD)
	FetchRecentRefsDays int
	// Makes the FetchRecentRefsDays option apply to remote refs from fetch source as well (default true)
	FetchRecentRefsIncludeRemotes bool
	// number of days prior to latest commit on a ref that we'll fetch previous
	// LFS changes too (default 0 = only fetch at ref)
	FetchRecentCommitsDays int
	// Whether to always fetch recent even without --recent
	FetchRecentAlways bool
	// Number of days added to FetchRecent*; data outside combined window will be
	// deleted when prune is run. (default 3)
	PruneOffsetDays int
	// Always verify with remote before pruning
	PruneVerifyRemoteAlways bool
	// Name of remote to check for unpushed and verify checks
	PruneRemoteName string
}

type Configuration struct {
	CurrentRemote         string
	httpClient            *HttpClient
	redirectingHttpClient *http.Client
	ntlmSession           ntlm.ClientSession
	envVars               map[string]string
	isTracingHttp         bool
	isLoggingStats        bool

	loading           sync.Mutex // guards initialization of gitConfig and remotes
	gitConfig         map[string]string
	origConfig        map[string]string
	remotes           []string
	extensions        map[string]Extension
	fetchIncludePaths []string
	fetchExcludePaths []string
	fetchPruneConfig  *FetchPruneConfig
}

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		envVars:       make(map[string]string),
	}
	c.isTracingHttp = c.GetenvBool("GIT_CURL_VERBOSE", false)
	c.isLoggingStats = c.GetenvBool("GIT_LOG_STATS", false)
	return c
}

func (c *Configuration) Getenv(key string) string {
	if i, ok := c.envVars[key]; ok {
		return i
	}

	v := os.Getenv(key)
	c.envVars[key] = v
	return v
}

func (c *Configuration) Setenv(key, value string) error {
	// Check see if we have this in our cache, if so update it
	if _, ok := c.envVars[key]; ok {
		c.envVars[key] = value
	}

	// Now set in process
	return os.Setenv(key, value)
}

// GetenvBool parses a boolean environment variable and returns the result as a bool.
// If the environment variable is unset, empty, or if the parsing fails,
// the value of def (default) is returned instead.
func (c *Configuration) GetenvBool(key string, def bool) bool {
	s := c.Getenv(key)
	if len(s) == 0 {
		return def
	}

	b, err := parseConfigBool(s)
	if err != nil {
		return def
	}
	return b
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return NewEndpoint(url)
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	if c.NtlmAccess() {
		return 1
	}

	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	value, ok := c.GitConfig("lfs.batch")
	if !ok || len(value) == 0 {
		return true
	}

	useBatch, err := parseConfigBool(value)
	if err != nil {
		return false
	}

	return useBatch
}

func (c *Configuration) NtlmAccess() bool {
	return c.Access() == "ntlm"
}

// PrivateAccess will retrieve the access value and return true if
// the value is set to private. When a repo is marked as having private
// access, the http requests for the batch api will fetch the credentials
// before running, otherwise the request will run without credentials.
func (c *Configuration) PrivateAccess() bool {
	return c.Access() != "none"
}

// Access returns the access auth type.
func (c *Configuration) Access() string {
	return c.EndpointAccess(c.Endpoint())
}

// SetAccess will set the private access flag in .git/config.
func (c *Configuration) SetAccess(authType string) {
	c.SetEndpointAccess(c.Endpoint(), authType)
}

func (c *Configuration) EndpointAccess(e Endpoint) string {
	key := fmt.Sprintf("lfs.%s.access", e.Url)
	if v, ok := c.GitConfig(key); ok && len(v) > 0 {
		lower := strings.ToLower(v)
		if lower == "private" {
			return "basic"
		}
		return lower
	}
	return "none"
}

func (c *Configuration) SetEndpointAccess(e Endpoint, authType string) {
	tracerx.Printf("setting repository access to %s", authType)
	key := fmt.Sprintf("lfs.%s.access", e.Url)

	// Modify the config cache because it's checked again in this process
	// without being reloaded.
	switch authType {
	case "", "none":
		git.Config.UnsetLocalKey("", key)

		c.loading.Lock()
		delete(c.gitConfig, strings.ToLower(key))
		c.loading.Unlock()
	default:
		git.Config.SetLocal("", key, authType)

		c.loading.Lock()
		c.gitConfig[strings.ToLower(key)] = authType
		c.loading.Unlock()
	}
}

func (c *Configuration) FetchIncludePaths() []string {
	c.loadGitConfig()
	return c.fetchIncludePaths
}
func (c *Configuration) FetchExcludePaths() []string {
	c.loadGitConfig()
	return c.fetchExcludePaths
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}
	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return NewEndpoint(url)
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		return NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) Extensions() map[string]Extension {
	c.loadGitConfig()
	return c.extensions
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) AllGitConfig() map[string]string {
	c.loadGitConfig()
	return c.gitConfig
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) FetchPruneConfig() *FetchPruneConfig {
	if c.fetchPruneConfig == nil {
		c.fetchPruneConfig = &FetchPruneConfig{
			FetchRecentRefsDays:           7,
			FetchRecentRefsIncludeRemotes: true,
			FetchRecentCommitsDays:        0,
			PruneOffsetDays:               3,
			PruneVerifyRemoteAlways:       false,
			PruneRemoteName:               "origin",
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentRefsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentremoterefs"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = b
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentcommitsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentCommitsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentalways"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentAlways = b
			}
		}
		if v, ok := c.GitConfig("lfs.pruneoffsetdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.PruneOffsetDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.pruneverifyremotealways"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.PruneVerifyRemoteAlways = b
			}
		}
		if v, ok := c.GitConfig("lfs.pruneremotetocheck"); ok {
			c.fetchPruneConfig.PruneRemoteName = v
		}

	}
	return c.fetchPruneConfig
}

func parseConfigBool(str string) (bool, error) {
	switch strings.ToLower(str) {
	case "true", "1", "on", "yes", "t":
		return true, nil
	case "false", "0", "off", "no", "f":
		return false, nil
	}
	return false, fmt.Errorf("Unable to parse %q as a boolean", str)
}

func (c *Configuration) loadGitConfig() bool {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return false
	}

	c.gitConfig = make(map[string]string)
	c.extensions = make(map[string]Extension)
	uniqRemotes := make(map[string]bool)

	configFiles := []string{
		filepath.Join(LocalWorkingDir, ".lfsconfig"),
		filepath.Join(LocalWorkingDir, ".gitconfig"),
	}
	c.readGitConfigFromFiles(configFiles, 0, uniqRemotes)

	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	c.readGitConfig(listOutput, uniqRemotes, false)

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}

	return true
}

func (c *Configuration) readGitConfigFromFiles(filenames []string, filenameIndex int, uniqRemotes map[string]bool) {
	filename := filenames[filenameIndex]
	_, err := os.Stat(filename)
	if err == nil {
		fileOutput, err := git.Config.ListFromFile(filename)
		if err != nil {
			panic(fmt.Errorf("Error listing git config from %s: %s", filename, err))
		}
		c.readGitConfig(fileOutput, uniqRemotes, true)
		return
	}

	if os.IsNotExist(err) {
		newIndex := filenameIndex + 1
		if len(filenames) > newIndex {
			c.readGitConfigFromFiles(filenames, newIndex, uniqRemotes)
		}
		return
	}

	panic(fmt.Errorf("Error listing git config from %s: %s", filename, err))
}

func (c *Configuration) readGitConfig(output string, uniqRemotes map[string]bool, onlySafe bool) {
	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}

		allowed := !onlySafe
		key := strings.ToLower(pieces[0])
		value := pieces[1]

		keyParts := strings.Split(key, ".")
		if len(keyParts) == 4 && keyParts[0] == "lfs" && keyParts[1] == "extension" {
			name := keyParts[2]
			ext := c.extensions[name]
			switch keyParts[3] {
			case "clean":
				if onlySafe {
					continue
				}
				ext.Clean = value
			case "smudge":
				if onlySafe {
					continue
				}
				ext.Smudge = value
			case "priority":
				allowed = true
				p, err := strconv.Atoi(value)
				if err == nil && p >= 0 {
					ext.Priority = p
				}
			}

			ext.Name = name
			c.extensions[name] = ext
		} else if len(keyParts) > 1 && keyParts[0] == "remote" {
			if onlySafe && (len(keyParts) == 3 && keyParts[2] != "lfsurl") {
				continue
			}

			allowed = true
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		}

		if !allowed && keyIsUnsafe(key) {
			continue
		}

		c.gitConfig[key] = value

		if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchinclude" {
			for _, inc := range strings.Split(value, ",") {
				inc = strings.TrimSpace(inc)
				c.fetchIncludePaths = append(c.fetchIncludePaths, inc)
			}
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchexclude" {
			for _, ex := range strings.Split(value, ",") {
				ex = strings.TrimSpace(ex)
				c.fetchExcludePaths = append(c.fetchExcludePaths, ex)
			}
		}
	}
}

func keyIsUnsafe(key string) bool {
	for _, safe := range safeKeys {
		if safe == key {
			return false
		}
	}
	return true
}

var safeKeys = []string{
	"lfs.url",
	"lfs.fetchinclude",
	"lfs.fetchexclude",
}
<commit_msg>ンンン ンンン ンン<commit_after>package lfs

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/ThomsonReutersEikon/go-ntlm/ntlm"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

var (
	Config        = NewConfig()
	defaultRemote = "origin"
)

// FetchPruneConfig collects together the config options that control fetching and pruning
type FetchPruneConfig struct {
	// The number of days prior to current date for which (local) refs other than HEAD
	// will be fetched with --recent (default 7, 0 = only fetch HEAD)
	FetchRecentRefsDays int
	// Makes the FetchRecentRefsDays option apply to remote refs from fetch source as well (default true)
	FetchRecentRefsIncludeRemotes bool
	// number of days prior to latest commit on a ref that we'll fetch previous
	// LFS changes too (default 0 = only fetch at ref)
	FetchRecentCommitsDays int
	// Whether to always fetch recent even without --recent
	FetchRecentAlways bool
	// Number of days added to FetchRecent*; data outside combined window will be
	// deleted when prune is run. (default 3)
	PruneOffsetDays int
	// Always verify with remote before pruning
	PruneVerifyRemoteAlways bool
	// Name of remote to check for unpushed and verify checks
	PruneRemoteName string
}

type Configuration struct {
	CurrentRemote         string
	httpClient            *HttpClient
	redirectingHttpClient *http.Client
	ntlmSession           ntlm.ClientSession
	envVars               map[string]string
	isTracingHttp         bool
	isLoggingStats        bool

	loading           sync.Mutex // guards initialization of gitConfig and remotes
	gitConfig         map[string]string
	origConfig        map[string]string
	remotes           []string
	extensions        map[string]Extension
	fetchIncludePaths []string
	fetchExcludePaths []string
	fetchPruneConfig  *FetchPruneConfig
}

func NewConfig() *Configuration {
	c := &Configuration{
		CurrentRemote: defaultRemote,
		envVars:       make(map[string]string),
	}
	c.isTracingHttp = c.GetenvBool("GIT_CURL_VERBOSE", false)
	c.isLoggingStats = c.GetenvBool("GIT_LOG_STATS", false)
	return c
}

func (c *Configuration) Getenv(key string) string {
	if i, ok := c.envVars[key]; ok {
		return i
	}

	v := os.Getenv(key)
	c.envVars[key] = v
	return v
}

func (c *Configuration) Setenv(key, value string) error {
	// Check see if we have this in our cache, if so update it
	if _, ok := c.envVars[key]; ok {
		c.envVars[key] = value
	}

	// Now set in process
	return os.Setenv(key, value)
}

// GetenvBool parses a boolean environment variable and returns the result as a bool.
// If the environment variable is unset, empty, or if the parsing fails,
// the value of def (default) is returned instead.
func (c *Configuration) GetenvBool(key string, def bool) bool {
	s := c.Getenv(key)
	if len(s) == 0 {
		return def
	}

	b, err := parseConfigBool(s)
	if err != nil {
		return def
	}
	return b
}

func (c *Configuration) Endpoint() Endpoint {
	if url, ok := c.GitConfig("lfs.url"); ok {
		return NewEndpoint(url)
	}

	if len(c.CurrentRemote) > 0 && c.CurrentRemote != defaultRemote {
		if endpoint := c.RemoteEndpoint(c.CurrentRemote); len(endpoint.Url) > 0 {
			return endpoint
		}
	}

	return c.RemoteEndpoint(defaultRemote)
}

func (c *Configuration) ConcurrentTransfers() int {
	if c.NtlmAccess() {
		return 1
	}

	uploads := 3

	if v, ok := c.GitConfig("lfs.concurrenttransfers"); ok {
		n, err := strconv.Atoi(v)
		if err == nil && n > 0 {
			uploads = n
		}
	}

	return uploads
}

func (c *Configuration) BatchTransfer() bool {
	value, ok := c.GitConfig("lfs.batch")
	if !ok || len(value) == 0 {
		return true
	}

	useBatch, err := parseConfigBool(value)
	if err != nil {
		return false
	}

	return useBatch
}

func (c *Configuration) NtlmAccess() bool {
	return c.Access() == "ntlm"
}

// PrivateAccess will retrieve the access value and return true if
// the value is set to private. When a repo is marked as having private
// access, the http requests for the batch api will fetch the credentials
// before running, otherwise the request will run without credentials.
func (c *Configuration) PrivateAccess() bool {
	return c.Access() != "none"
}

// Access returns the access auth type.
func (c *Configuration) Access() string {
	return c.EndpointAccess(c.Endpoint())
}

// SetAccess will set the private access flag in .git/config.
func (c *Configuration) SetAccess(authType string) {
	c.SetEndpointAccess(c.Endpoint(), authType)
}

func (c *Configuration) EndpointAccess(e Endpoint) string {
	key := fmt.Sprintf("lfs.%s.access", e.Url)
	if v, ok := c.GitConfig(key); ok && len(v) > 0 {
		lower := strings.ToLower(v)
		if lower == "private" {
			return "basic"
		}
		return lower
	}
	return "none"
}

func (c *Configuration) SetEndpointAccess(e Endpoint, authType string) {
	tracerx.Printf("setting repository access to %s", authType)
	key := fmt.Sprintf("lfs.%s.access", e.Url)

	// Modify the config cache because it's checked again in this process
	// without being reloaded.
	switch authType {
	case "", "none":
		git.Config.UnsetLocalKey("", key)

		c.loading.Lock()
		delete(c.gitConfig, strings.ToLower(key))
		c.loading.Unlock()
	default:
		git.Config.SetLocal("", key, authType)

		c.loading.Lock()
		c.gitConfig[strings.ToLower(key)] = authType
		c.loading.Unlock()
	}
}

func (c *Configuration) FetchIncludePaths() []string {
	c.loadGitConfig()
	return c.fetchIncludePaths
}
func (c *Configuration) FetchExcludePaths() []string {
	c.loadGitConfig()
	return c.fetchExcludePaths
}

func (c *Configuration) RemoteEndpoint(remote string) Endpoint {
	if len(remote) == 0 {
		remote = defaultRemote
	}
	if url, ok := c.GitConfig("remote." + remote + ".lfsurl"); ok {
		return NewEndpoint(url)
	}

	if url, ok := c.GitConfig("remote." + remote + ".url"); ok {
		return NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (c *Configuration) Remotes() []string {
	c.loadGitConfig()
	return c.remotes
}

func (c *Configuration) Extensions() map[string]Extension {
	c.loadGitConfig()
	return c.extensions
}

func (c *Configuration) GitConfig(key string) (string, bool) {
	c.loadGitConfig()
	value, ok := c.gitConfig[strings.ToLower(key)]
	return value, ok
}

func (c *Configuration) AllGitConfig() map[string]string {
	c.loadGitConfig()
	return c.gitConfig
}

func (c *Configuration) ObjectUrl(oid string) (*url.URL, error) {
	return ObjectUrl(c.Endpoint(), oid)
}

func (c *Configuration) FetchPruneConfig() *FetchPruneConfig {
	if c.fetchPruneConfig == nil {
		c.fetchPruneConfig = &FetchPruneConfig{
			FetchRecentRefsDays:           7,
			FetchRecentRefsIncludeRemotes: true,
			FetchRecentCommitsDays:        0,
			PruneOffsetDays:               3,
			PruneVerifyRemoteAlways:       false,
			PruneRemoteName:               "origin",
		}
		if v, ok := c.GitConfig("lfs.fetchrecentrefsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentRefsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentremoterefs"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentRefsIncludeRemotes = b
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentcommitsdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.FetchRecentCommitsDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.fetchrecentalways"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.FetchRecentAlways = b
			}
		}
		if v, ok := c.GitConfig("lfs.pruneoffsetdays"); ok {
			n, err := strconv.Atoi(v)
			if err == nil && n >= 0 {
				c.fetchPruneConfig.PruneOffsetDays = n
			}
		}
		if v, ok := c.GitConfig("lfs.pruneverifyremotealways"); ok {
			if b, err := parseConfigBool(v); err == nil {
				c.fetchPruneConfig.PruneVerifyRemoteAlways = b
			}
		}
		if v, ok := c.GitConfig("lfs.pruneremotetocheck"); ok {
			c.fetchPruneConfig.PruneRemoteName = v
		}

	}
	return c.fetchPruneConfig
}

func parseConfigBool(str string) (bool, error) {
	switch strings.ToLower(str) {
	case "true", "1", "on", "yes", "t":
		return true, nil
	case "false", "0", "off", "no", "f":
		return false, nil
	}
	return false, fmt.Errorf("Unable to parse %q as a boolean", str)
}

func (c *Configuration) loadGitConfig() bool {
	c.loading.Lock()
	defer c.loading.Unlock()

	if c.gitConfig != nil {
		return false
	}

	c.gitConfig = make(map[string]string)
	c.extensions = make(map[string]Extension)
	uniqRemotes := make(map[string]bool)

	configFiles := []string{
		filepath.Join(LocalWorkingDir, ".lfsconfig"),

		// TODO: remove .gitconfig support for Git LFS v2.0
		filepath.Join(LocalWorkingDir, ".gitconfig"),
	}
	c.readGitConfigFromFiles(configFiles, 0, uniqRemotes)

	listOutput, err := git.Config.List()
	if err != nil {
		panic(fmt.Errorf("Error listing git config: %s", err))
	}

	c.readGitConfig(listOutput, uniqRemotes, false)

	c.remotes = make([]string, 0, len(uniqRemotes))
	for remote, isOrigin := range uniqRemotes {
		if isOrigin {
			continue
		}
		c.remotes = append(c.remotes, remote)
	}

	return true
}

func (c *Configuration) readGitConfigFromFiles(filenames []string, filenameIndex int, uniqRemotes map[string]bool) {
	filename := filenames[filenameIndex]
	_, err := os.Stat(filename)
	if err == nil {
		fileOutput, err := git.Config.ListFromFile(filename)
		if err != nil {
			panic(fmt.Errorf("Error listing git config from %s: %s", filename, err))
		}
		c.readGitConfig(fileOutput, uniqRemotes, true)
		return
	}

	if os.IsNotExist(err) {
		newIndex := filenameIndex + 1
		if len(filenames) > newIndex {
			expected := ".lfsconfig"
			fmt.Fprintf(os.Stderr, "WARNING: Reading LFS config from %q, not %q. Rename to %q before Git LFS v2.0 to remove this warning.\n",
				filepath.Base(filenames[newIndex]), expected, expected)
			c.readGitConfigFromFiles(filenames, newIndex, uniqRemotes)
		}
		return
	}

	panic(fmt.Errorf("Error listing git config from %s: %s", filename, err))
}

func (c *Configuration) readGitConfig(output string, uniqRemotes map[string]bool, onlySafe bool) {
	lines := strings.Split(output, "\n")
	for _, line := range lines {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}

		allowed := !onlySafe
		key := strings.ToLower(pieces[0])
		value := pieces[1]

		keyParts := strings.Split(key, ".")
		if len(keyParts) == 4 && keyParts[0] == "lfs" && keyParts[1] == "extension" {
			name := keyParts[2]
			ext := c.extensions[name]
			switch keyParts[3] {
			case "clean":
				if onlySafe {
					continue
				}
				ext.Clean = value
			case "smudge":
				if onlySafe {
					continue
				}
				ext.Smudge = value
			case "priority":
				allowed = true
				p, err := strconv.Atoi(value)
				if err == nil && p >= 0 {
					ext.Priority = p
				}
			}

			ext.Name = name
			c.extensions[name] = ext
		} else if len(keyParts) > 1 && keyParts[0] == "remote" {
			if onlySafe && (len(keyParts) == 3 && keyParts[2] != "lfsurl") {
				continue
			}

			allowed = true
			remote := keyParts[1]
			uniqRemotes[remote] = remote == "origin"
		}

		if !allowed && keyIsUnsafe(key) {
			continue
		}

		c.gitConfig[key] = value

		if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchinclude" {
			for _, inc := range strings.Split(value, ",") {
				inc = strings.TrimSpace(inc)
				c.fetchIncludePaths = append(c.fetchIncludePaths, inc)
			}
		} else if len(keyParts) == 2 && keyParts[0] == "lfs" && keyParts[1] == "fetchexclude" {
			for _, ex := range strings.Split(value, ",") {
				ex = strings.TrimSpace(ex)
				c.fetchExcludePaths = append(c.fetchExcludePaths, ex)
			}
		}
	}
}

func keyIsUnsafe(key string) bool {
	for _, safe := range safeKeys {
		if safe == key {
			return false
		}
	}
	return true
}

var safeKeys = []string{
	"lfs.url",
	"lfs.fetchinclude",
	"lfs.fetchexclude",
}
<|endoftext|>


<commit_before>package main

import "github.com/zserge/webview"

func main() {
	webview.Open("Hello", "https://golang.org", 800, 600, true)
}
<commit_msg>make a more self-contained golang example<commit_after>package main

import (
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"os/user"

	"github.com/zserge/webview"
)

var tmpl = template.Must(template.New("").Parse(`

	  <form action="/exit">

Hello, {{ .Name }}!

			<input type="submit" value="Exit" />
		</form>

`))

func main() {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}
	defer ln.Close()

	go func() {
		http.HandleFunc("/exit", func(w http.ResponseWriter, r *http.Request) {
			os.Exit(0)
		})
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			u, _ := user.Current()
			tmpl.Execute(w, u)
		})
		log.Fatal(http.Serve(ln, nil))
	}()

	webview.Open("Hello", "http://"+ln.Addr().String(), 400, 300, false)
}
<|endoftext|>


<commit_before>// Package crd handles Managed Prometheus Custom Resource Definitions.
package crd

import (
	"fmt"
	"strings"

	"go.skia.org/infra/go/prom"
)

const notInClustersAnnotationKey = "not-in-clusters"

// Rules Custom Resource representation.
//
// In theory we should be able to import this from the Managed Prometheus repo, but
// they don't currently provide a separate repo with just the json annotated structs.
type Rules struct {
	Version  string   `yaml:"apiVersion"`
	Kind     string   `yaml:"kind"`
	MetaData MetaData `yaml:"metadata"`
	Spec     Spec     `yaml:"spec"`
}

// MetaData for the Rules CRD.
type MetaData struct {
	Name      string `yaml:"name"`
	Namespace string `yaml:"namespace,omitempty"`
}

// Spec for parsing the yaml format of Prometheus alerts.
type Spec struct {
	Groups []Group `yaml:"groups"`
}

// Group of Rules.
type Group struct {
	Name     string `yaml:"name"`
	Interval string `yaml:"interval"`
	Rules    []Rule `yaml:"rules"`
}

// Rule is a single Prometheus Alert.
type Rule struct {
	Alert       string            `yaml:"alert"`
	Expr        string            `yaml:"expr"`
	Labels      map[string]string `yaml:"labels"`
	Annotations map[string]string `yaml:"annotations"`
}

// Skip returns true if the alert should not be applied to the given cluster.
func (r Rule) Skip(cluster string) bool {
	if skipString, ok := r.Annotations[notInClustersAnnotationKey]; ok {
		for _, skipCluster := range strings.Split(skipString, ",") {
			if cluster == strings.TrimSpace(skipCluster) {
				return true
			}
		}
	}
	return false
}

// AddAbsentRules adds an `absent()` alert for each Rule, where possible.
func (r *Rules) AddAbsentRules(cluster string) {
	absentGroups := []Group{}
	for _, g := range r.Spec.Groups {
		rules := []Rule{}
		for _, rule := range g.Rules {
			if rule.Skip(cluster) {
				continue
			}
			equation, ignore := prom.EquationFromExpr(rule.Expr)
			if ignore {
				continue
			}
			rules = append(rules, Rule{
				Alert: "Absent",
				Expr:  fmt.Sprintf("absent(%s)", equation),
				Labels: map[string]string{
					"category": "infra",
					"severify": "critical",
				},
				Annotations: map[string]string{
					"abbr":        rule.Alert,
					"equation":    equation,
					"description": fmt.Sprintf("There is no data for the Alert: %q", rule.Alert),
				},
			})
		}
		absentGroups = append(absentGroups, Group{
			Name:     fmt.Sprintf("absent-%s", g.Name),
			Interval: g.Interval,
			Rules:    rules,
		})
	}
	r.Spec.Groups = append(r.Spec.Groups, absentGroups...)
}
<commit_msg>Change not-in-clusters to not_in_clusters.<commit_after>// Package crd handles Managed Prometheus Custom Resource Definitions.
package crd

import (
	"fmt"
	"strings"

	"go.skia.org/infra/go/prom"
)

const notInClustersAnnotationKey = "not_in_clusters"

// Rules Custom Resource representation.
//
// In theory we should be able to import this from the Managed Prometheus repo, but
// they don't currently provide a separate repo with just the json annotated structs.
type Rules struct {
	Version  string   `yaml:"apiVersion"`
	Kind     string   `yaml:"kind"`
	MetaData MetaData `yaml:"metadata"`
	Spec     Spec     `yaml:"spec"`
}

// MetaData for the Rules CRD.
type MetaData struct {
	Name      string `yaml:"name"`
	Namespace string `yaml:"namespace,omitempty"`
}

// Spec for parsing the yaml format of Prometheus alerts.
type Spec struct {
	Groups []Group `yaml:"groups"`
}

// Group of Rules.
type Group struct {
	Name     string `yaml:"name"`
	Interval string `yaml:"interval"`
	Rules    []Rule `yaml:"rules"`
}

// Rule is a single Prometheus Alert.
type Rule struct {
	Alert       string            `yaml:"alert"`
	Expr        string            `yaml:"expr"`
	Labels      map[string]string `yaml:"labels"`
	Annotations map[string]string `yaml:"annotations"`
}

// Skip returns true if the alert should not be applied to the given cluster.
func (r Rule) Skip(cluster string) bool {
	if skipString, ok := r.Annotations[notInClustersAnnotationKey]; ok {
		for _, skipCluster := range strings.Split(skipString, ",") {
			if cluster == strings.TrimSpace(skipCluster) {
				return true
			}
		}
	}
	return false
}

// AddAbsentRules adds an `absent()` alert for each Rule, where possible.
func (r *Rules) AddAbsentRules(cluster string) {
	absentGroups := []Group{}
	for _, g := range r.Spec.Groups {
		rules := []Rule{}
		for _, rule := range g.Rules {
			if rule.Skip(cluster) {
				continue
			}
			equation, ignore := prom.EquationFromExpr(rule.Expr)
			if ignore {
				continue
			}
			rules = append(rules, Rule{
				Alert: "Absent",
				Expr:  fmt.Sprintf("absent(%s)", equation),
				Labels: map[string]string{
					"category": "infra",
					"severify": "critical",
				},
				Annotations: map[string]string{
					"abbr":        rule.Alert,
					"equation":    equation,
					"description": fmt.Sprintf("There is no data for the Alert: %q", rule.Alert),
				},
			})
		}
		absentGroups = append(absentGroups, Group{
			Name:     fmt.Sprintf("absent-%s", g.Name),
			Interval: g.Interval,
			Rules:    rules,
		})
	}
	r.Spec.Groups = append(r.Spec.Groups, absentGroups...)
}
<|endoftext|>


<commit_before>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync2

import (
	"sync"
	"sync/atomic"
	"time"
)

// AtomicInt32 is a wrapper with a simpler interface around atomic.(Add|Store|Load|CompareAndSwap)Int32 functions.
type AtomicInt32 struct {
	int32
}

// NewAtomicInt32 initializes a new AtomicInt32 with a given value.
func NewAtomicInt32(n int32) AtomicInt32 {
	return AtomicInt32{n}
}

// Add atomically adds n to the value.
func (i *AtomicInt32) Add(n int32) int32 {
	return atomic.AddInt32(&i.int32, n)
}

// Set atomically sets n as new value.
func (i *AtomicInt32) Set(n int32) {
	atomic.StoreInt32(&i.int32, n)
}

// Get atomically returns the current value.
func (i *AtomicInt32) Get() int32 {
	return atomic.LoadInt32(&i.int32)
}

// CompareAndSwap atomatically swaps the old with the new value.
func (i *AtomicInt32) CompareAndSwap(oldval, newval int32) (swapped bool) {
	return atomic.CompareAndSwapInt32(&i.int32, oldval, newval)
}

// AtomicInt64 is a wrapper with a simpler interface around atomic.(Add|Store|Load|CompareAndSwap)Int64 functions.
type AtomicInt64 struct {
	int64
}

// NewAtomicInt64 initializes a new AtomicInt64 with a given value.
func NewAtomicInt64(n int64) AtomicInt64 {
	return AtomicInt64{n}
}

// Add atomically adds n to the value.
func (i *AtomicInt64) Add(n int64) int64 {
	return atomic.AddInt64(&i.int64, n)
}

// Set atomically sets n as new value.
func (i *AtomicInt64) Set(n int64) {
	atomic.StoreInt64(&i.int64, n)
}

// Get atomically returns the current value.
func (i *AtomicInt64) Get() int64 {
	return atomic.LoadInt64(&i.int64)
}

// CompareAndSwap atomatically swaps the old with the new value.
func (i *AtomicInt64) CompareAndSwap(oldval, newval int64) (swapped bool) {
	return atomic.CompareAndSwapInt64(&i.int64, oldval, newval)
}

// AtomicDuration is a wrapper with a simpler interface around atomic.(Add|Store|Load|CompareAndSwap)Int64 functions.
type AtomicDuration struct {
	int64
}

// NewAtomicDuration initializes a new AtomicDuration with a given value.
func NewAtomicDuration(duration time.Duration) AtomicDuration {
	return AtomicDuration{int64(duration)}
}

// Add atomically adds duration to the value.
func (d *AtomicDuration) Add(duration time.Duration) time.Duration {
	return time.Duration(atomic.AddInt64(&d.int64, int64(duration)))
}

// Set atomically sets duration as new value.
func (d *AtomicDuration) Set(duration time.Duration) {
	atomic.StoreInt64(&d.int64, int64(duration))
}

// Get atomically returns the current value.
func (d *AtomicDuration) Get() time.Duration {
	return time.Duration(atomic.LoadInt64(&d.int64))
}

// CompareAndSwap atomatically swaps the old with the new value.
func (d *AtomicDuration) CompareAndSwap(oldval, newval time.Duration) (swapped bool) {
	return atomic.CompareAndSwapInt64(&d.int64, int64(oldval), int64(newval))
}

// AtomicString gives you atomic-style APIs for string, but
// it's only a convenience wrapper that uses a mutex. So, it's
// not as efficient as the rest of the atomic types.
type AtomicString struct {
	mu  sync.Mutex
	str string
}

func (s *AtomicString) Set(str string) {
	s.mu.Lock()
	s.str = str
	s.mu.Unlock()
}

func (s *AtomicString) Get() string {
	s.mu.Lock()
	str := s.str
	s.mu.Unlock()
	return str
}

func (s *AtomicString) CompareAndSwap(oldval, newval string) (swqpped bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.str == oldval {
		s.str = newval
		return true
	}
	return false
}
<commit_msg>Fix golint warnings for AtomicString.<commit_after>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync2

import (
	"sync"
	"sync/atomic"
	"time"
)

// AtomicInt32 is a wrapper with a simpler interface around atomic.(Add|Store|Load|CompareAndSwap)Int32 functions.
type AtomicInt32 struct {
	int32
}

// NewAtomicInt32 initializes a new AtomicInt32 with a given value.
func NewAtomicInt32(n int32) AtomicInt32 {
	return AtomicInt32{n}
}

// Add atomically adds n to the value.
func (i *AtomicInt32) Add(n int32) int32 {
	return atomic.AddInt32(&i.int32, n)
}

// Set atomically sets n as new value.
func (i *AtomicInt32) Set(n int32) {
	atomic.StoreInt32(&i.int32, n)
}

// Get atomically returns the current value.
func (i *AtomicInt32) Get() int32 {
	return atomic.LoadInt32(&i.int32)
}

// CompareAndSwap atomatically swaps the old with the new value.
func (i *AtomicInt32) CompareAndSwap(oldval, newval int32) (swapped bool) {
	return atomic.CompareAndSwapInt32(&i.int32, oldval, newval)
}

// AtomicInt64 is a wrapper with a simpler interface around atomic.(Add|Store|Load|CompareAndSwap)Int64 functions.
type AtomicInt64 struct {
	int64
}

// NewAtomicInt64 initializes a new AtomicInt64 with a given value.
func NewAtomicInt64(n int64) AtomicInt64 {
	return AtomicInt64{n}
}

// Add atomically adds n to the value.
func (i *AtomicInt64) Add(n int64) int64 {
	return atomic.AddInt64(&i.int64, n)
}

// Set atomically sets n as new value.
func (i *AtomicInt64) Set(n int64) {
	atomic.StoreInt64(&i.int64, n)
}

// Get atomically returns the current value.
func (i *AtomicInt64) Get() int64 {
	return atomic.LoadInt64(&i.int64)
}

// CompareAndSwap atomatically swaps the old with the new value.
func (i *AtomicInt64) CompareAndSwap(oldval, newval int64) (swapped bool) {
	return atomic.CompareAndSwapInt64(&i.int64, oldval, newval)
}

// AtomicDuration is a wrapper with a simpler interface around atomic.(Add|Store|Load|CompareAndSwap)Int64 functions.
type AtomicDuration struct {
	int64
}

// NewAtomicDuration initializes a new AtomicDuration with a given value.
func NewAtomicDuration(duration time.Duration) AtomicDuration {
	return AtomicDuration{int64(duration)}
}

// Add atomically adds duration to the value.
func (d *AtomicDuration) Add(duration time.Duration) time.Duration {
	return time.Duration(atomic.AddInt64(&d.int64, int64(duration)))
}

// Set atomically sets duration as new value.
func (d *AtomicDuration) Set(duration time.Duration) {
	atomic.StoreInt64(&d.int64, int64(duration))
}

// Get atomically returns the current value.
func (d *AtomicDuration) Get() time.Duration {
	return time.Duration(atomic.LoadInt64(&d.int64))
}

// CompareAndSwap atomatically swaps the old with the new value.
func (d *AtomicDuration) CompareAndSwap(oldval, newval time.Duration) (swapped bool) {
	return atomic.CompareAndSwapInt64(&d.int64, int64(oldval), int64(newval))
}

// AtomicString gives you atomic-style APIs for string, but
// it's only a convenience wrapper that uses a mutex. So, it's
// not as efficient as the rest of the atomic types.
type AtomicString struct {
	mu  sync.Mutex
	str string
}

// Set atomically sets str as new value.
func (s *AtomicString) Set(str string) {
	s.mu.Lock()
	s.str = str
	s.mu.Unlock()
}

// Get atomically returns the current value.
func (s *AtomicString) Get() string {
	s.mu.Lock()
	str := s.str
	s.mu.Unlock()
	return str
}

// CompareAndSwap atomatically swaps the old with the new value.
func (s *AtomicString) CompareAndSwap(oldval, newval string) (swqpped bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.str == oldval {
		s.str = newval
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"testing"

	"k8s.io/apimachinery/pkg/util/intstr"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/test"
)

// TestIngressTLS verifies that the Ingress properly handles the TLS field.
func TestIngressTLS(t *testing.T) {
	t.Skip("https://github.com/knative/serving/issues/6401")
	t.Parallel()
	clients := test.Setup(t)

	name, port, cancel := CreateRuntimeService(t, clients, networking.ServicePortNameHTTP1)
	defer cancel()

	hosts := []string{name + ".example.com"}

	secretName, cancel := CreateTLSSecret(t, clients, hosts)
	defer cancel()

	_, client, cancel := CreateIngressReady(t, clients, v1alpha1.IngressSpec{
		Rules: []v1alpha1.IngressRule{{
			Hosts:      hosts,
			Visibility: v1alpha1.IngressVisibilityExternalIP,
			HTTP: &v1alpha1.HTTPIngressRuleValue{
				Paths: []v1alpha1.HTTPIngressPath{{
					Splits: []v1alpha1.IngressBackendSplit{{
						IngressBackend: v1alpha1.IngressBackend{
							ServiceName:      name,
							ServiceNamespace: test.ServingNamespace,
							ServicePort:      intstr.FromInt(port),
						},
					}},
				}},
			},
		}},
		TLS: []v1alpha1.IngressTLS{{
			Hosts:           hosts,
			SecretName:      secretName,
			SecretNamespace: test.ServingNamespace,
		}},
	})
	defer cancel()

	t.Run("verify HTTP", func(t *testing.T) {
		RuntimeRequest(t, client, "http://"+name+".example.com")
	})

	t.Run("verify HTTPS", func(t *testing.T) {
		RuntimeRequest(t, client, "https://"+name+".example.com")
	})
}

// TODO(mattmoor): Consider adding variants where we have multiple hosts with distinct certificates.
<commit_msg>Reinstate IngressTLS tests. (#6420)<commit_after>// +build e2e

/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"testing"

	"k8s.io/apimachinery/pkg/util/intstr"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/test"
)

// TestIngressTLS verifies that the Ingress properly handles the TLS field.
func TestIngressTLS(t *testing.T) {
	t.Parallel()
	clients := test.Setup(t)

	name, port, cancel := CreateRuntimeService(t, clients, networking.ServicePortNameHTTP1)
	defer cancel()

	hosts := []string{name + ".example.com"}

	secretName, cancel := CreateTLSSecret(t, clients, hosts)
	defer cancel()

	_, client, cancel := CreateIngressReady(t, clients, v1alpha1.IngressSpec{
		Rules: []v1alpha1.IngressRule{{
			Hosts:      hosts,
			Visibility: v1alpha1.IngressVisibilityExternalIP,
			HTTP: &v1alpha1.HTTPIngressRuleValue{
				Paths: []v1alpha1.HTTPIngressPath{{
					Splits: []v1alpha1.IngressBackendSplit{{
						IngressBackend: v1alpha1.IngressBackend{
							ServiceName:      name,
							ServiceNamespace: test.ServingNamespace,
							ServicePort:      intstr.FromInt(port),
						},
					}},
				}},
			},
		}},
		TLS: []v1alpha1.IngressTLS{{
			Hosts:           hosts,
			SecretName:      secretName,
			SecretNamespace: test.ServingNamespace,
		}},
	})
	defer cancel()

	t.Run("verify HTTP", func(t *testing.T) {
		RuntimeRequest(t, client, "http://"+name+".example.com")
	})

	t.Run("verify HTTPS", func(t *testing.T) {
		RuntimeRequest(t, client, "https://"+name+".example.com")
	})
}

// TODO(mattmoor): Consider adding variants where we have multiple hosts with distinct certificates.
<|endoftext|>