<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"io"
	"math"
	"strings"
	"testing"
	"time"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestStattingObjectSyncer(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const srcObjectContents = "taco"

type StattingObjectSyncerTest struct {
	ctx context.Context

	bucket gcs.Bucket
	leaser lease.FileLeaser
	syncer ObjectSyncer
	clock  timeutil.SimulatedClock

	srcObject *gcs.Object
	content   mutable.Content
}

var _ SetUpInterface = &StattingObjectSyncerTest{}

func init() { RegisterTestSuite(&StattingObjectSyncerTest{}) }

func (t *StattingObjectSyncerTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Set up dependencies.
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")
	t.leaser = lease.NewFileLeaser("", math.MaxInt32, math.MaxInt32)
	t.syncer = createStattingObjectSyncer(
		t.serveSyncFull,
		t.serveSyncAppend)

	t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local))

	// Set up a source object.
	t.srcObject, err = t.bucket.CreateObject(
		t.ctx,
		&gcs.CreateObjectRequest{
			Name:     "foo",
			Contents: strings.NewReader(srcObjectContents),
		})

	AssertEq(nil, err)

	// Wrap a mutable.Content around it.
	t.content = mutable.NewContent(
		NewReadProxy(
			t.srcObject,
			nil,            // Initial read lease
			math.MaxUint64, // Chunk size
			t.leaser,
			t.bucket),
		&t.clock)
}

func (t *StattingObjectSyncerTest) call() (
	rl lease.ReadLease, o *gcs.Object, err error) {
	err = errors.New("TODO")
	return
}

func (t *StattingObjectSyncerTest) serveSyncFull(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	err = errors.New("TODO: serveSyncFull")
	return
}

func (t *StattingObjectSyncerTest) serveSyncAppend(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	err = errors.New("TODO: serveSyncAppend")
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *StattingObjectSyncerTest) NotDirty() {
	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(nil, rl)
	ExpectEq(nil, o)
}

func (t *StattingObjectSyncerTest) SmallerThanSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SameSizeAsSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) LargerThanSource_ThresholdInSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) LargerThanSource_ThresholdAtEndOfSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncFullFails() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncFullSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncAppendFails() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncAppendSucceeds() {
	AssertTrue(false, "TODO")
}
<commit_msg>Fixed build errors.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"io"
	"io/ioutil"
	"math"
	"strings"
	"testing"
	"time"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/mutable"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestStattingObjectSyncer(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// fakeObjectCreator
////////////////////////////////////////////////////////////////////////

// An objectCreator that records the arguments it is called with, returning
// canned results.
type fakeObjectCreator struct {
	called bool

	// Supplied arguments
	srcObject *gcs.Object
	contents  []byte

	// Canned results
	o   *gcs.Object
	err error
}

func (oc *fakeObjectCreator) Create(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	// Have we been called more than once?
	AssertFalse(oc.called)
	oc.called = true

	// Record args.
	oc.srcObject = srcObject
	oc.contents, err = ioutil.ReadAll(r)
	AssertEq(nil, err)

	// Return results.
	o, err = oc.o, oc.err
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const srcObjectContents = "taco"

type StattingObjectSyncerTest struct {
	ctx context.Context

	fullCreator   fakeObjectCreator
	appendCreator fakeObjectCreator

	bucket gcs.Bucket
	leaser lease.FileLeaser
	syncer ObjectSyncer
	clock  timeutil.SimulatedClock

	srcObject *gcs.Object
	content   mutable.Content
}

var _ SetUpInterface = &StattingObjectSyncerTest{}

func init() { RegisterTestSuite(&StattingObjectSyncerTest{}) }

func (t *StattingObjectSyncerTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Set up dependencies.
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")
	t.leaser = lease.NewFileLeaser("", math.MaxInt32, math.MaxInt32)
	t.syncer = createStattingObjectSyncer(&t.fullCreator, &t.appendCreator)
	t.clock.SetTime(time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local))

	// Set up a source object.
	t.srcObject, err = t.bucket.CreateObject(
		t.ctx,
		&gcs.CreateObjectRequest{
			Name:     "foo",
			Contents: strings.NewReader(srcObjectContents),
		})

	AssertEq(nil, err)

	// Wrap a mutable.Content around it.
	t.content = mutable.NewContent(
		NewReadProxy(
			t.srcObject,
			nil,            // Initial read lease
			math.MaxUint64, // Chunk size
			t.leaser,
			t.bucket),
		&t.clock)
}

func (t *StattingObjectSyncerTest) call() (
	rl lease.ReadLease, o *gcs.Object, err error) {
	err = errors.New("TODO")
	return
}

func (t *StattingObjectSyncerTest) serveSyncFull(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	err = errors.New("TODO: serveSyncFull")
	return
}

func (t *StattingObjectSyncerTest) serveSyncAppend(
	ctx context.Context,
	srcObject *gcs.Object,
	r io.Reader) (o *gcs.Object, err error) {
	err = errors.New("TODO: serveSyncAppend")
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *StattingObjectSyncerTest) NotDirty() {
	// Call
	rl, o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(nil, rl)
	ExpectEq(nil, o)
}

func (t *StattingObjectSyncerTest) SmallerThanSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SameSizeAsSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) LargerThanSource_ThresholdInSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) LargerThanSource_ThresholdAtEndOfSource() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncFullFails() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncFullSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncAppendFails() {
	AssertTrue(false, "TODO")
}

func (t *StattingObjectSyncerTest) SyncAppendSucceeds() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////
// Purpose: Turn the duoyinzi-phrase.txt file to json as Go dictionary
// Authors: Tong Sun (c) 2017
// Sources: https://github.com/mozillazg/phrase-pinyin-data
////////////////////////////////////////////////////////////////////////////

package main

import (
	"bytes"
	"fmt"
	"os"
	"strings"

	"github.com/spakin/awk"
)

// json string to return
var bufRet = bytes.NewBufferString("")

// var p = print X: use of builtin print not in function call
// Remove "//D>" to debug

func main() {
	// https://godoc.org/github.com/spakin/awk
	s := awk.NewScript()

	// == BEGIN
	sa := s.NewValueArray()
	// last word, and its length
	sa.Set("wLast", "")
	sa.Set("cLength", 0)
	bufRet.WriteByte('{')

	// == Match & Process
	s.AppendStmt(nil, func(s *awk.Script) {
		ww, py := s.F(1).String(), ""
		for ii := 2; ii <= s.NF; ii++ {
			py += s.F(ii).String() + " "
		}
		ww = ww[:len(ww)-1] // last char is ":"
		//D>print(s.NR, ww, py)

		// count of current word length and match length with last word
		cLength := sa.Get("cLength").Int()
		cMatch := commPrefixLen(sa.Get("wLast").String(), ww)
		//D>print(" ", cLength, " ", cMatch)

		lDiff := cLength - cMatch
		if lDiff == 0 && sa.Get("wLast").String() != "" {
			// the new phrase is longer than last one, ignore it
			//D>println()
			s.Next()
		}

		sa.Set("wLast", ww)
		//print(" Saved:", sa.Get("wLast").String())
		// Only partial match, close the json, with (lDiff-1) closes
		for ii := 1; ii < lDiff; ii++ {
			fmt.Fprintf(bufRet, "},")
		}
		outputEntry(ww, py, cMatch)
		sa.Set("cLength", len([]rune(ww)))
	})

	// == END
	s.End = func(s *awk.Script) {
		for ii := 0; ii < sa.Get("cLength").Int(); ii++ {
			bufRet.WriteByte('}')
		}
		//ret := bufRet.String()
		ret := strings.Replace(bufRet.String(), ",}", "}", -1)
		fmt.Printf("%s\n", ret)
	}

	if err := s.Run(os.Stdin); err != nil {
		panic(err)
	}
}

func outputEntry(s, py string, start int) {
	//D>println(" output ", s, " from ", start)
	rs := []rune(s)

	for ii := start; ii < len(rs)-1; ii++ {
		fmt.Fprintf(bufRet, `"%s":{`, string(rs[ii]))
	}
	fmt.Fprintf(bufRet, `"%s":"%s",`, string(rs[len(rs)-1]), py)
}

func commPrefixLen(s1, s2 string) int {
	//D>print(" compare ", s1, ":", s2)
	rs1, rs2 := []rune(s1), []rune(s2)
	ii := 0
	for ; ii < min(len(rs1), len(rs2)); ii++ {
		if rs1[ii] != rs2[ii] {
			return ii
		}
	}
	return ii
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}

func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func debug(args ...interface{}) {
	// print(args...) X: invalid use of ... with builtin print
}

/*
 */
<commit_msg>- [*] better way to toggle the printing on and off<commit_after>////////////////////////////////////////////////////////////////////////////
// Purpose: Turn the duoyinzi-phrase.txt file to json as Go dictionary
// Authors: Tong Sun (c) 2017
// Sources: https://github.com/mozillazg/phrase-pinyin-data
////////////////////////////////////////////////////////////////////////////

package main

import (
	"bytes"
	"fmt"
	"os"
	"strings"

	"github.com/spakin/awk"
)

// json string to return
var bufRet = bytes.NewBufferString("")

func main() {
	// https://godoc.org/github.com/spakin/awk
	s := awk.NewScript()

	// == BEGIN
	sa := s.NewValueArray()
	// last word, and its length
	sa.Set("wLast", "")
	sa.Set("cLength", 0)
	bufRet.WriteByte('{')

	// == Match & Process
	s.AppendStmt(nil, func(s *awk.Script) {
		ww, py := s.F(1).String(), ""
		for ii := 2; ii <= s.NF; ii++ {
			py += s.F(ii).String() + " "
		}
		ww = ww[:len(ww)-1] // last char is ":"
		print(s.NR, ww, py)

		// count of current word length and match length with last word
		cLength := sa.Get("cLength").Int()
		cMatch := commPrefixLen(sa.Get("wLast").String(), ww)
		print(" ", cLength, " ", cMatch)

		lDiff := cLength - cMatch
		if lDiff == 0 && sa.Get("wLast").String() != "" {
			// the new phrase is longer than last one, ignore it
			print("\n")
			s.Next()
		}

		sa.Set("wLast", ww)
		//print(" Saved:", sa.Get("wLast").String())
		// Only partial match, close the json, with (lDiff-1) closes
		for ii := 1; ii < lDiff; ii++ {
			fmt.Fprintf(bufRet, "},")
		}
		outputEntry(ww, py, cMatch)
		sa.Set("cLength", len([]rune(ww)))
	})

	// == END
	s.End = func(s *awk.Script) {
		for ii := 0; ii < sa.Get("cLength").Int(); ii++ {
			bufRet.WriteByte('}')
		}
		//ret := bufRet.String()
		ret := strings.Replace(bufRet.String(), ",}", "}", -1)
		fmt.Printf("%s\n", ret)
	}

	if err := s.Run(os.Stdin); err != nil {
		panic(err)
	}
}

func outputEntry(s, py string, start int) {
	print(" output ", s, " from ", start, "\n")
	rs := []rune(s)

	for ii := start; ii < len(rs)-1; ii++ {
		fmt.Fprintf(bufRet, `"%s":{`, string(rs[ii]))
	}
	fmt.Fprintf(bufRet, `"%s":"%s",`, string(rs[len(rs)-1]), py)
}

func commPrefixLen(s1, s2 string) int {
	print(" compare ", s1, ":", s2)
	rs1, rs2 := []rune(s1), []rune(s2)
	ii := 0
	for ; ii < min(len(rs1), len(rs2)); ii++ {
		if rs1[ii] != rs2[ii] {
			return ii
		}
	}
	return ii
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}

func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

// Comment this out to debug
func print(args ...interface{}) {}

// Kevin Malachowski
// https://groups.google.com/d/msg/golang-nuts/ycfywzTAADY/5fchXEo1BgAJ
<|endoftext|>


<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package thrift

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

const DEFAULT_MAX_LENGTH = 16384000

type FramedTransport struct {
	transport Transport
	buf       bytes.Buffer  // buffers the writes
	reader    *bufio.Reader // just a buffer over the underlying transport
	frameSize uint32        //Current remaining size of the frame. if ==0 read next frame header
	rBuffer   [4]byte       // used for reading
	wBuffer   [4]byte       // used for writing
	maxLength uint32
}

type framedTransportFactory struct {
	factory   TransportFactory
	maxLength uint32
}

func NewFramedTransportFactory(factory TransportFactory) TransportFactory {
	return &framedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
}

func NewFramedTransportFactoryMaxLength(factory TransportFactory, maxLength uint32) TransportFactory {
	return &framedTransportFactory{factory: factory, maxLength: maxLength}
}

func (p *framedTransportFactory) GetTransport(base Transport) Transport {
	return NewFramedTransportMaxLength(p.factory.GetTransport(base), p.maxLength)
}

func NewFramedTransport(transport Transport) *FramedTransport {
	return &FramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
}

func NewFramedTransportMaxLength(transport Transport, maxLength uint32) *FramedTransport {
	return &FramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
}

func (p *FramedTransport) Open() error {
	return p.transport.Open()
}

func (p *FramedTransport) IsOpen() bool {
	return p.transport.IsOpen()
}

func (p *FramedTransport) Close() error {
	return p.transport.Close()
}

func (p *FramedTransport) Read(buf []byte) (l int, err error) {
	if p.frameSize == 0 {
		p.frameSize, err = p.readFrameHeader()
		if err != nil {
			return
		}
	}
	if p.frameSize < uint32(len(buf)) {
		frameSize := p.frameSize
		tmp := make([]byte, p.frameSize)
		l, err = p.Read(tmp)
		copy(buf, tmp)
		if err == nil {
			err = NewTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", frameSize, len(buf)))
			return
		}
	}
	got, err := p.reader.Read(buf)
	p.frameSize = p.frameSize - uint32(got)
	//sanity check
	if p.frameSize < 0 {
		return 0, NewTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "Negative frame size")
	}
	return got, NewTransportExceptionFromError(err)
}

func (p *FramedTransport) ReadByte() (c byte, err error) {
	if p.frameSize == 0 {
		p.frameSize, err = p.readFrameHeader()
		if err != nil {
			return
		}
	}
	if p.frameSize < 1 {
		return 0, NewTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", p.frameSize, 1))
	}
	c, err = p.reader.ReadByte()
	if err == nil {
		p.frameSize--
	}
	return
}

func (p *FramedTransport) Write(buf []byte) (int, error) {
	n, err := p.buf.Write(buf)
	return n, NewTransportExceptionFromError(err)
}

func (p *FramedTransport) WriteByte(c byte) error {
	return p.buf.WriteByte(c)
}

func (p *FramedTransport) WriteString(s string) (n int, err error) {
	return p.buf.WriteString(s)
}

func (p *FramedTransport) Flush() error {
	size := p.buf.Len()
	buf := p.wBuffer[:4]
	binary.BigEndian.PutUint32(buf, uint32(size))
	_, err := p.transport.Write(buf)
	if err != nil {
		return NewTransportExceptionFromError(err)
	}
	if size > 0 {
		if n, err := p.buf.WriteTo(p.transport); err != nil {
			print("Error while flushing write buffer of size ", size, " to transport, only wrote ", n, " bytes: ", err.Error(), "\n")
			return NewTransportExceptionFromError(err)
		}
	}
	err = p.transport.Flush()
	return NewTransportExceptionFromError(err)
}

func (p *FramedTransport) readFrameHeader() (uint32, error) {
	buf := p.rBuffer[:4]
	if _, err := io.ReadFull(p.reader, buf); err != nil {
		return 0, err
	}
	size := binary.BigEndian.Uint32(buf)
	if size < 0 || size > p.maxLength {
		return 0, NewTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
	}
	return size, nil
}

func (p *FramedTransport) RemainingBytes() (num_bytes uint64) {
	return uint64(p.frameSize)
}
<commit_msg>Make framed transport pre-read the frame<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package thrift

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
)

const DEFAULT_MAX_LENGTH = 16384000

type FramedTransport struct {
	transport Transport
	framebuf  byteReader    // buffer for reading complete frames off the wire
	buf       bytes.Buffer  // buffers the writes
	reader    *bufio.Reader // just a buffer over the underlying transport
	frameSize uint32        // Current remaining size of the frame. if ==0 read next frame header
	rBuffer   [4]byte       // used for reading
	wBuffer   [4]byte       // used for writing
	maxLength uint32
}

type framedTransportFactory struct {
	factory   TransportFactory
	maxLength uint32
}

func NewFramedTransportFactory(factory TransportFactory) TransportFactory {
	return &framedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
}

func NewFramedTransportFactoryMaxLength(factory TransportFactory, maxLength uint32) TransportFactory {
	return &framedTransportFactory{factory: factory, maxLength: maxLength}
}

func (p *framedTransportFactory) GetTransport(base Transport) Transport {
	return NewFramedTransportMaxLength(p.factory.GetTransport(base), p.maxLength)
}

func NewFramedTransport(transport Transport) *FramedTransport {
	return &FramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
}

func NewFramedTransportMaxLength(transport Transport, maxLength uint32) *FramedTransport {
	return &FramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
}

func (p *FramedTransport) Open() error {
	return p.transport.Open()
}

func (p *FramedTransport) IsOpen() bool {
	return p.transport.IsOpen()
}

func (p *FramedTransport) Close() error {
	return p.transport.Close()
}

func (p *FramedTransport) Read(buf []byte) (l int, err error) {
	if p.frameSize == 0 {
		p.frameSize, err = p.readFrameHeader()
		if err != nil {
			return
		}
	}
	if p.frameSize < uint32(len(buf)) {
		frameSize := p.frameSize
		tmp := make([]byte, p.frameSize)
		l, err = p.Read(tmp)
		copy(buf, tmp)
		if err == nil {
			err = NewTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", frameSize, len(buf)))
			return
		}
	}
	got, err := p.framebuf.Read(buf)
	p.frameSize = p.frameSize - uint32(got)
	//sanity check
	if p.frameSize < 0 {
		return 0, NewTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "Negative frame size")
	}
	return got, NewTransportExceptionFromError(err)
}

func (p *FramedTransport) ReadByte() (c byte, err error) {
	if p.frameSize == 0 {
		p.frameSize, err = p.readFrameHeader()
		if err != nil {
			return
		}
	}
	if p.frameSize < 1 {
		return 0, NewTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", p.frameSize, 1))
	}
	c, err = p.framebuf.ReadByte()
	if err == nil {
		p.frameSize--
	}
	return
}

func (p *FramedTransport) Write(buf []byte) (int, error) {
	n, err := p.buf.Write(buf)
	return n, NewTransportExceptionFromError(err)
}

func (p *FramedTransport) WriteByte(c byte) error {
	return p.buf.WriteByte(c)
}

func (p *FramedTransport) WriteString(s string) (n int, err error) {
	return p.buf.WriteString(s)
}

func (p *FramedTransport) Flush() error {
	size := p.buf.Len()
	buf := p.wBuffer[:4]
	binary.BigEndian.PutUint32(buf, uint32(size))
	_, err := p.transport.Write(buf)
	if err != nil {
		return NewTransportExceptionFromError(err)
	}
	if size > 0 {
		if n, err := p.buf.WriteTo(p.transport); err != nil {
			print("Error while flushing write buffer of size ", size, " to transport, only wrote ", n, " bytes: ", err.Error(), "\n")
			return NewTransportExceptionFromError(err)
		}
	}
	err = p.transport.Flush()
	return NewTransportExceptionFromError(err)
}

func (p *FramedTransport) readFrameHeader() (uint32, error) {
	buf := p.rBuffer[:4]
	if _, err := io.ReadFull(p.reader, buf); err != nil {
		return 0, err
	}
	size := binary.BigEndian.Uint32(buf)
	if size < 0 || size > p.maxLength {
		return 0, NewTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
	}

	framebuf := newLimitedByteReader(p.reader, int64(size))
	out, err := ioutil.ReadAll(framebuf)
	if err != nil {
		return 0, err
	}
	if uint32(len(out)) < size {
		return 0, NewTransportExceptionFromError(fmt.Errorf("Unable to read full frame of size %d", size))
	}
	p.framebuf = newLimitedByteReader(bytes.NewBuffer(out), int64(size))

	return size, nil
}

func (p *FramedTransport) RemainingBytes() (num_bytes uint64) {
	return uint64(p.frameSize)
}
<|endoftext|>


<commit_before>package ui

import (
	"fmt"
	"regexp"
	"strconv"

	"github.com/atotto/clipboard"
	"github.com/tcolar/goed/core"
)

func (v *View) ClearSelections() {
	v.selections = []core.Selection{}
}

// Text returns the text contained in the selection of the given view
// Note: **NOT** a rectangle but from pt1 to pt2
func (v *View) SelectionText(s *core.Selection) [][]rune {
	cf := s.ColFrom
	ct := s.ColTo
	lt := s.LineTo
	lf := s.LineFrom
	if lf == lt {
		return *v.backend.Slice(lf, cf, lt, ct).Text()
	}
	// first line
	text := *v.backend.Slice(lf, cf, lf, -1).Text()
	for l := lf + 1; l < lt; l++ {
		// middle
		text = append(text, *v.backend.Slice(l, 1, l, -1).Text()...)
	}
	// last line
	text = append(text, *v.backend.Slice(lt, 1, lt, ct).Text()...)
	return text
}

// Selected returns whether the text at line, col is current selected
// also returns the matching selection, if any.
func (v *View) Selected(col, line int) (bool, *core.Selection) {
	for _, s := range v.selections {
		if line < s.LineFrom || line > s.LineTo {
			continue
		} else if line > s.LineFrom && line < s.LineTo {
			return true, &s
		} else if s.LineFrom == s.LineTo {
			return col >= s.ColFrom && col <= s.ColTo, &s
		} else if line == s.LineFrom && col >= s.ColFrom {
			return true, &s
		} else if line == s.LineTo && col <= s.ColTo {
			return true, &s
		}
	}
	return false, nil
}

func (v *View) SelectionCopy(s *core.Selection) {
	t := v.SelectionText(s)
	core.Ed.SetStatus(fmt.Sprintf("Copied %d lines to clipboard.", len(t)))
	clipboard.WriteAll(core.RunesToString(t))
}

func (v *View) SelectionDelete(s *core.Selection) {
	v.Delete(s.LineFrom-1, s.ColFrom-1, s.LineTo-1, s.ColTo-1)
}

func (v *View) Paste() {
	text, err := clipboard.ReadAll()
	if err != nil {
		core.Ed.SetStatusErr(err.Error())
		return
	}
	if len(v.selections) > 0 {
		v.DeleteCur()
	}
	_, x, y := v.CurChar()
	v.Insert(y, x, text)
}

var locationRegexp = regexp.MustCompile(`([^"\s(){}[\]<>,?|+=&^%#@!;':]+)(:\d+)?(:\d+)?`)

// Try to select a "location" from the given position
// a location is a path with possibly a line number and maybe a column number as well
func (v *View) PathSelection(line, col int) *core.Selection {
	l := v.Line(v.slice, line-1)
	ln := string(l)
	slice := core.NewSlice(1, 1, 1, len(l)+1, [][]rune{l})
	c := v.lineRunesTo(slice, 0, col)
	matches := locationRegexp.FindAllStringIndex(ln, -1)
	var best []int
	// Find the "narrowest" match around the cursor
	for _, s := range matches {
		if s[0] <= c && s[1] >= c {
			if best == nil || s[1]-s[0] < best[1]-best[0] {
				best = s
			}
		}
	}
	if best == nil {
		return nil
	}
	// TODO: if a path like a go import, try to find that path up from curdir ?
	return core.NewSelection(line, best[0]+1, line, best[1])
}

// Parses a selection into a location (file, line, col)
func (v *View) SelectionToLoc(sel *core.Selection) (loc string, line, col int) {
	sub := locationRegexp.FindAllStringSubmatch(core.RunesToString(v.SelectionText(sel)), 1)
	if len(sub) == 0 {
		return
	}
	s := sub[0]
	if len(s) >= 1 {
		loc = s[1]
	}
	if len(s[2]) > 0 {
		line, _ = strconv.Atoi(s[2][1:])
	}
	if len(s[3]) > 0 {
		col, _ = strconv.Atoi(s[3][1:])
	}
	return loc, line, col
}

// Expand a selection toward a new position
func (v *View) ExpandSelection(prevl, prevc, l, c int) {
	if len(v.selections) == 0 {
		s := *core.NewSelection(prevl, prevc, l, c)
		v.selections = []core.Selection{
			s,
		}
	} else {
		s := v.selections[0]
		if s.LineTo == prevl && s.ColTo == prevc {
			s.LineTo, s.ColTo = l, c
		} else {
			s.LineFrom, s.ColFrom = l, c
		}
		s.Normalize()
		v.selections[0] = s
	}
}
<commit_msg>Select word on double click<commit_after>package ui

import (
	"fmt"
	"regexp"
	"strconv"
	"unicode"

	"github.com/atotto/clipboard"
	"github.com/tcolar/goed/core"
)

func (v *View) ClearSelections() {
	v.selections = []core.Selection{}
}

// Text returns the text contained in the selection of the given view
// Note: **NOT** a rectangle but from pt1 to pt2
func (v *View) SelectionText(s *core.Selection) [][]rune {
	cf := s.ColFrom
	ct := s.ColTo
	lt := s.LineTo
	lf := s.LineFrom
	if lf == lt {
		return *v.backend.Slice(lf, cf, lt, ct).Text()
	}
	// first line
	text := *v.backend.Slice(lf, cf, lf, -1).Text()
	for l := lf + 1; l < lt; l++ {
		// middle
		text = append(text, *v.backend.Slice(l, 0, l, -1).Text()...)
	}
	// last line
	text = append(text, *v.backend.Slice(lt, 0, lt, ct).Text()...)
	return text
}

// Selected returns whether the text at line, col is current selected
// also returns the matching selection, if any.
func (v *View) Selected(col, line int) (bool, *core.Selection) {
	for _, s := range v.selections {
		if line < s.LineFrom || line > s.LineTo {
			continue
		} else if line > s.LineFrom && line < s.LineTo {
			return true, &s
		} else if s.LineFrom == s.LineTo {
			return col >= s.ColFrom && col <= s.ColTo, &s
		} else if line == s.LineFrom && col >= s.ColFrom {
			return true, &s
		} else if line == s.LineTo && col <= s.ColTo {
			return true, &s
		}
	}
	return false, nil
}

func (v *View) SelectionCopy(s *core.Selection) {
	t := v.SelectionText(s)
	core.Ed.SetStatus(fmt.Sprintf("Copied %d lines to clipboard.", len(t)))
	clipboard.WriteAll(core.RunesToString(t))
}

func (v *View) SelectionDelete(s *core.Selection) {
	y, x := s.LineFrom-v.CurLine(), s.ColFrom-v.CurCol()
	v.Delete(s.LineFrom, s.ColFrom, s.LineTo, s.ColTo)
	v.MoveCursor(x, y)
}

func (v *View) Paste() {
	text, err := clipboard.ReadAll()
	if err != nil {
		core.Ed.SetStatusErr(err.Error())
		return
	}
	if len(v.selections) > 0 {
		v.DeleteCur()
	}
	_, x, y := v.CurChar()
	v.Insert(y, x, text)
}

var locationRegexp = regexp.MustCompile(`([^"\s(){}[\]<>,?|+=&^%#@!;':]+)(:\d+)?(:\d+)?`)

// Try to select a "location" from the given position
// a location is a path with possibly a line number and maybe a column number as well
func (v *View) ExpandSelectionPath(line, col int) *core.Selection {
	l := v.Line(v.slice, line)
	ln := string(l)
	slice := core.NewSlice(0, 0, 0, len(l), [][]rune{l})
	c := v.lineRunesTo(slice, 0, col)
	matches := locationRegexp.FindAllStringIndex(ln, -1)
	var best []int
	// Find the "narrowest" match around the cursor
	for _, s := range matches {
		if s[0] <= c && s[1] >= c {
			if best == nil || s[1]-s[0] < best[1]-best[0] {
				best = s
			}
		}
	}
	if best == nil {
		return nil
	}
	return core.NewSelection(line, best[0], line, best[1]-1)
}

// Try to select the longest "word" from current position.
func (v *View) ExpandSelectionWord(line, col int) *core.Selection {
	l := v.Line(v.slice, line)
	c := v.lineRunesTo(v.slice, line, col)
	c1, c2 := c, c
	for ; c1 >= 0 && isWordRune(l[c1]); c1-- {
	}
	c1++
	for ; c2 < len(l) && isWordRune(l[c2]); c2++ {
	}
	c2--
	if c1 >= c2 {
		return nil
	}
	return core.NewSelection(line, c1, line, c2)
}

func isWordRune(r rune) bool {
	return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_'
}

// Select the whole given line
func (v *View) SelectLine(line int) {
	s := core.NewSelection(line, 0, line, v.LineLen(v.slice, line))
	v.selections = []core.Selection{
		*s,
	}
}

// Select a word at the given location (if any)
func (v *View) SelectWord(line, col int) {
	s := v.ExpandSelectionWord(line, col)
	if s != nil {
		v.selections = []core.Selection{
			*s,
		}
	}
}

// Parses a selection into a location (file, line, col)
func (v *View) SelectionToLoc(sel *core.Selection) (loc string, line, col int) {
	sub := locationRegexp.FindAllStringSubmatch(core.RunesToString(v.SelectionText(sel)), 1)
	if len(sub) == 0 {
		return
	}
	s := sub[0]
	if len(s) >= 1 {
		loc = s[1]
	}
	if len(s[2]) > 0 {
		line, _ = strconv.Atoi(s[2][1:])
	}
	if len(s[3]) > 0 {
		col, _ = strconv.Atoi(s[3][1:])
	}
	return loc, line, col
}

// Stretch a selection toward a new position
func (v *View) StretchSelection(prevl, prevc, l, c int) {
	if len(v.selections) == 0 {
		s := *core.NewSelection(prevl, prevc, l, c)
		v.selections = []core.Selection{
			s,
		}
	} else {
		s := v.selections[0]
		if s.LineTo == prevl && s.ColTo == prevc {
			s.LineTo, s.ColTo = l, c
		} else {
			s.LineFrom, s.ColFrom = l, c
		}
		s.Normalize()
		v.selections[0] = s
	}
}

// Open what's selected or under the cursor
// if newView is true then open in a new view, otherwise
// replace content of v
func (v *View) OpenSelection(newView bool) {
	ed := core.Ed.(*Editor)
	newView = newView || v.Dirty()
	if len(v.selections) == 0 {
		selection := v.ExpandSelectionPath(v.CurLine(), v.CurCol())
		if selection == nil {
			ed.SetStatusErr("Could not expand location from cursor location.")
			return
		}
		v.selections = []core.Selection{*selection}
	}
	loc, line, col := v.SelectionToLoc(&v.selections[0])
	isDir := false
	loc, isDir = core.LookupLocation(v.WorkDir(), loc)
	vv := ed.ViewByLoc(loc)
	if vv != nil {
		// Already open
		ed.ActivateView(vv.(*View), col, line)
		return
	}
	v2 := ed.NewView()
	if _, err := ed.Open(loc, v2, v.WorkDir()); err != nil {
		ed.SetStatusErr(err.Error())
		return
	}
	if newView {
		if isDir {
			ed.InsertView(v2, v, 0.5)
		} else {
			ed.InsertViewSmart(v2)
		}
	} else {
		ed.ReplaceView(v, v2)
	}
	ed.ActivateView(v2, col, line)
}
<|endoftext|>


<commit_before>// Package cloudflare is a store implementation backed by cloudflare workers kv
// Note that the cloudflare workers KV API is eventually consistent.
package cloudflare

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/micro/go-micro/v2/store"
	"github.com/pkg/errors"

	"github.com/patrickmn/go-cache"
)

const (
	apiBaseURL = "https://api.cloudflare.com/client/v4/"
)

type workersKV struct {
	options store.Options
	// cf account id
	account string
	// cf api token
	token string
	// cf kv namespace
	namespace string
	// http client to use
	httpClient *http.Client
	// cache
	cache *cache.Cache
}

// apiResponse is a cloudflare v4 api response
type apiResponse struct {
	Result []struct {
		ID         string    `json:"id"`
		Type       string    `json:"type"`
		Name       string    `json:"name"`
		Expiration string    `json:"expiration"`
		Content    string    `json:"content"`
		Proxiable  bool      `json:"proxiable"`
		Proxied    bool      `json:"proxied"`
		TTL        int       `json:"ttl"`
		Priority   int       `json:"priority"`
		Locked     bool      `json:"locked"`
		ZoneID     string    `json:"zone_id"`
		ZoneName   string    `json:"zone_name"`
		ModifiedOn time.Time `json:"modified_on"`
		CreatedOn  time.Time `json:"created_on"`
	} `json:"result"`
	Success bool         `json:"success"`
	Errors  []apiMessage `json:"errors"`
	// not sure Messages is ever populated?
	Messages   []apiMessage `json:"messages"`
	ResultInfo struct {
		Page       int `json:"page"`
		PerPage    int `json:"per_page"`
		Count      int `json:"count"`
		TotalCount int `json:"total_count"`
	} `json:"result_info"`
}

// apiMessage is a Cloudflare v4 API Error
type apiMessage struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// getOptions returns account id, token and namespace
func getOptions() (string, string, string) {
	accountID := os.Getenv("CF_ACCOUNT_ID")
	apiToken := os.Getenv("CF_API_TOKEN")
	namespace := os.Getenv("KV_NAMESPACE_ID")

	return accountID, apiToken, namespace
}

func validateOptions(account, token, namespace string) {
	if len(account) == 0 {
		log.Fatal("Store: CF_ACCOUNT_ID is blank")
	}

	if len(token) == 0 {
		log.Fatal("Store: CF_API_TOKEN is blank")
	}

	if len(namespace) == 0 {
		log.Fatal("Store: KV_NAMESPACE_ID is blank")
	}
}

func (w *workersKV) Init(opts ...store.Option) error {
	for _, o := range opts {
		o(&w.options)
	}
	if len(w.options.Namespace) > 0 {
		w.namespace = w.options.Namespace
	}
	ttl := w.options.Context.Value("STORE_CACHE_TTL")
	if ttl != nil {
		ttlduration, ok := ttl.(time.Duration)
		if !ok {
			log.Fatal("STORE_CACHE_TTL from context must be type int64")
		}
		w.cache = cache.New(ttlduration, 3*ttlduration)
	}
	return nil
}

func (w *workersKV) list(prefix string) ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/keys", w.account, w.namespace)

	body := make(map[string]string)

	if len(prefix) > 0 {
		body["prefix"] = prefix
	}

	response, _, _, err := w.request(ctx, http.MethodGet, path, body, make(http.Header))
	if err != nil {
		return nil, err
	}

	a := &apiResponse{}
	if err := json.Unmarshal(response, a); err != nil {
		return nil, err
	}

	if !a.Success {
		messages := ""
		for _, m := range a.Errors {
			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
		}
		return nil, errors.New(messages)
	}

	keys := make([]string, 0, len(a.Result))

	for _, r := range a.Result {
		keys = append(keys, r.Name)
	}

	return keys, nil
}

// In the cloudflare workers KV implemention, List() doesn't guarantee
// anything as the workers API is eventually consistent.
func (w *workersKV) List() ([]*store.Record, error) {
	keys, err := w.list("")
	if err != nil {
		return nil, err
	}

	var gerr error
	var records []*store.Record

	for _, key := range keys {
		r, err := w.Read(key)
		if err != nil {
			gerr = err
			continue
		}
		records = append(records, r...)
	}

	return records, gerr
}

func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	var options store.ReadOptions
	for _, o := range opts {
		o(&options)
	}

	keys := []string{key}

	if options.Prefix {
		k, err := w.list(key)
		if err != nil {
			return nil, err
		}
		keys = k
	}

	//nolint:prealloc
	var records []*store.Record

	for _, k := range keys {
		if w.cache != nil {
			if resp, hit := w.cache.Get(k); hit {
				if record, ok := resp.(*store.Record); ok {
					records = append(records, record)
					continue
				}
			}
		}

		path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k))
		response, headers, status, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header))
		if err != nil {
			return records, err
		}
		if status < 200 || status >= 300 {
			return records, errors.New("Received unexpected Status " + strconv.Itoa(status) + string(response))
		}
		record := &store.Record{
			Key:   k,
			Value: response,
		}
		if expiry := headers.Get("Expiration"); len(expiry) != 0 {
			expiryUnix, err := strconv.ParseInt(expiry, 10, 64)
			if err != nil {
				return records, err
			}
			record.Expiry = time.Until(time.Unix(expiryUnix, 0))
		}
		if w.cache != nil {
			w.cache.Set(record.Key, record, cache.DefaultExpiration)
		}
		records = append(records, record)
	}

	return records, nil
}

func (w *workersKV) Write(r *store.Record) error {
	// Set it in local cache, with the global TTL from options
	if w.cache != nil {
		w.cache.Set(r.Key, r, cache.DefaultExpiration)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(r.Key))
	if r.Expiry != 0 {
		// Minimum cloudflare TTL is 60 Seconds
		exp := int(math.Max(60, math.Round(r.Expiry.Seconds())))
		path = path + "?expiration_ttl=" + strconv.Itoa(exp)
	}

	headers := make(http.Header)

	resp, _, _, err := w.request(ctx, http.MethodPut, path, r.Value, headers)
	if err != nil {
		return err
	}

	a := &apiResponse{}
	if err := json.Unmarshal(resp, a); err != nil {
		return err
	}

	if !a.Success {
		messages := ""
		for _, m := range a.Errors {
			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
		}
		return errors.New(messages)
	}

	return nil
}

func (w *workersKV) Delete(key string) error {
	if w.cache != nil {
		w.cache.Delete(key)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(key))
	resp, _, _, err := w.request(ctx, http.MethodDelete, path, nil, make(http.Header))
	if err != nil {
		return err
	}

	a := &apiResponse{}
	if err := json.Unmarshal(resp, a); err != nil {
		return err
	}

	if !a.Success {
		messages := ""
		for _, m := range a.Errors {
			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
		}
		return errors.New(messages)
	}

	return nil
}

func (w *workersKV) request(ctx context.Context, method, path string, body interface{}, headers http.Header) ([]byte, http.Header, int, error) {
	var jsonBody []byte
	var err error

	if body != nil {
		if paramBytes, ok := body.([]byte); ok {
			jsonBody = paramBytes
		} else {
			jsonBody, err = json.Marshal(body)
			if err != nil {
				return nil, nil, 0, errors.Wrap(err, "error marshalling params to JSON")
			}
		}
	} else {
		jsonBody = nil
	}

	var reqBody io.Reader

	if jsonBody != nil {
		reqBody = bytes.NewReader(jsonBody)
	}

	req, err := http.NewRequestWithContext(ctx, method, apiBaseURL+path, reqBody)
	if err != nil {
		return nil, nil, 0, errors.Wrap(err, "error creating new request")
	}

	for key, value := range headers {
		req.Header[key] = value
	}

	// set token if it exists
	if len(w.token) > 0 {
		req.Header.Set("Authorization", "Bearer "+w.token)
	}

	// set the user agent to micro
	req.Header.Set("User-Agent", "micro/1.0 (https://micro.mu)")

	// Official cloudflare client does exponential backoff here
	// TODO: retry and use util/backoff
	resp, err := w.httpClient.Do(req)
	if err != nil {
		return nil, nil, 0, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return respBody, resp.Header, resp.StatusCode, err
	}

	return respBody, resp.Header, resp.StatusCode, nil
}

func (w *workersKV) String() string {
	return "cloudflare"
}

// NewStore returns a cloudflare Store implementation.
// Account ID, Token and Namespace must either be passed as options or
// environment variables. If set as env vars we expect the following;
// CF_API_TOKEN to a cloudflare API token scoped to Workers KV.
// CF_ACCOUNT_ID to contain a string with your cloudflare account ID.
// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage.
func NewStore(opts ...store.Option) store.Store {
	var options store.Options
	for _, o := range opts {
		o(&options)
	}

	// get options from environment
	account, token, namespace := getOptions()

	if len(account) == 0 {
		account = getAccount(options.Context)
	}

	if len(token) == 0 {
		token = getToken(options.Context)
	}

	if len(namespace) == 0 {
		namespace = options.Namespace
	}

	// validate options are not blank or log.Fatal
	validateOptions(account, token, namespace)

	return &workersKV{
		account:    account,
		namespace:  namespace,
		token:      token,
		options:    options,
		httpClient: &http.Client{},
	}
}
<commit_msg>Expiration is actually a unix timestamp (#1290)<commit_after>// Package cloudflare is a store implementation backed by cloudflare workers kv
// Note that the cloudflare workers KV API is eventually consistent.
package cloudflare

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/micro/go-micro/v2/store"
	"github.com/pkg/errors"

	"github.com/patrickmn/go-cache"
)

const (
	apiBaseURL = "https://api.cloudflare.com/client/v4/"
)

type workersKV struct {
	options store.Options
	// cf account id
	account string
	// cf api token
	token string
	// cf kv namespace
	namespace string
	// http client to use
	httpClient *http.Client
	// cache
	cache *cache.Cache
}

// apiResponse is a cloudflare v4 api response
type apiResponse struct {
	Result []struct {
		ID         string    `json:"id"`
		Type       string    `json:"type"`
		Name       string    `json:"name"`
		Expiration int64     `json:"expiration"`
		Content    string    `json:"content"`
		Proxiable  bool      `json:"proxiable"`
		Proxied    bool      `json:"proxied"`
		TTL        int64     `json:"ttl"`
		Priority   int64     `json:"priority"`
		Locked     bool      `json:"locked"`
		ZoneID     string    `json:"zone_id"`
		ZoneName   string    `json:"zone_name"`
		ModifiedOn time.Time `json:"modified_on"`
		CreatedOn  time.Time `json:"created_on"`
	} `json:"result"`
	Success bool         `json:"success"`
	Errors  []apiMessage `json:"errors"`
	// not sure Messages is ever populated?
	Messages   []apiMessage `json:"messages"`
	ResultInfo struct {
		Page       int `json:"page"`
		PerPage    int `json:"per_page"`
		Count      int `json:"count"`
		TotalCount int `json:"total_count"`
	} `json:"result_info"`
}

// apiMessage is a Cloudflare v4 API Error
type apiMessage struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// getOptions returns account id, token and namespace
func getOptions() (string, string, string) {
	accountID := os.Getenv("CF_ACCOUNT_ID")
	apiToken := os.Getenv("CF_API_TOKEN")
	namespace := os.Getenv("KV_NAMESPACE_ID")

	return accountID, apiToken, namespace
}

func validateOptions(account, token, namespace string) {
	if len(account) == 0 {
		log.Fatal("Store: CF_ACCOUNT_ID is blank")
	}

	if len(token) == 0 {
		log.Fatal("Store: CF_API_TOKEN is blank")
	}

	if len(namespace) == 0 {
		log.Fatal("Store: KV_NAMESPACE_ID is blank")
	}
}

func (w *workersKV) Init(opts ...store.Option) error {
	for _, o := range opts {
		o(&w.options)
	}
	if len(w.options.Namespace) > 0 {
		w.namespace = w.options.Namespace
	}
	ttl := w.options.Context.Value("STORE_CACHE_TTL")
	if ttl != nil {
		ttlduration, ok := ttl.(time.Duration)
		if !ok {
			log.Fatal("STORE_CACHE_TTL from context must be type int64")
		}
		w.cache = cache.New(ttlduration, 3*ttlduration)
	}
	return nil
}

func (w *workersKV) list(prefix string) ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/keys", w.account, w.namespace)

	body := make(map[string]string)

	if len(prefix) > 0 {
		body["prefix"] = prefix
	}

	response, _, _, err := w.request(ctx, http.MethodGet, path, body, make(http.Header))
	if err != nil {
		return nil, err
	}

	a := &apiResponse{}
	if err := json.Unmarshal(response, a); err != nil {
		return nil, err
	}

	if !a.Success {
		messages := ""
		for _, m := range a.Errors {
			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
		}
		return nil, errors.New(messages)
	}

	keys := make([]string, 0, len(a.Result))

	for _, r := range a.Result {
		keys = append(keys, r.Name)
	}

	return keys, nil
}

// In the cloudflare workers KV implemention, List() doesn't guarantee
// anything as the workers API is eventually consistent.
func (w *workersKV) List() ([]*store.Record, error) {
	keys, err := w.list("")
	if err != nil {
		return nil, err
	}

	var gerr error
	var records []*store.Record

	for _, key := range keys {
		r, err := w.Read(key)
		if err != nil {
			gerr = err
			continue
		}
		records = append(records, r...)
	}

	return records, gerr
}

func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	var options store.ReadOptions
	for _, o := range opts {
		o(&options)
	}

	keys := []string{key}

	if options.Prefix {
		k, err := w.list(key)
		if err != nil {
			return nil, err
		}
		keys = k
	}

	//nolint:prealloc
	var records []*store.Record

	for _, k := range keys {
		if w.cache != nil {
			if resp, hit := w.cache.Get(k); hit {
				if record, ok := resp.(*store.Record); ok {
					records = append(records, record)
					continue
				}
			}
		}

		path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k))
		response, headers, status, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header))
		if err != nil {
			return records, err
		}
		if status < 200 || status >= 300 {
			return records, errors.New("Received unexpected Status " + strconv.Itoa(status) + string(response))
		}
		record := &store.Record{
			Key:   k,
			Value: response,
		}
		if expiry := headers.Get("Expiration"); len(expiry) != 0 {
			expiryUnix, err := strconv.ParseInt(expiry, 10, 64)
			if err != nil {
				return records, err
			}
			record.Expiry = time.Until(time.Unix(expiryUnix, 0))
		}
		if w.cache != nil {
			w.cache.Set(record.Key, record, cache.DefaultExpiration)
		}
		records = append(records, record)
	}

	return records, nil
}

func (w *workersKV) Write(r *store.Record) error {
	// Set it in local cache, with the global TTL from options
	if w.cache != nil {
		w.cache.Set(r.Key, r, cache.DefaultExpiration)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(r.Key))
	if r.Expiry != 0 {
		// Minimum cloudflare TTL is 60 Seconds
		exp := int(math.Max(60, math.Round(r.Expiry.Seconds())))
		path = path + "?expiration_ttl=" + strconv.Itoa(exp)
	}

	headers := make(http.Header)

	resp, _, _, err := w.request(ctx, http.MethodPut, path, r.Value, headers)
	if err != nil {
		return err
	}

	a := &apiResponse{}
	if err := json.Unmarshal(resp, a); err != nil {
		return err
	}

	if !a.Success {
		messages := ""
		for _, m := range a.Errors {
			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
		}
		return errors.New(messages)
	}

	return nil
}

func (w *workersKV) Delete(key string) error {
	if w.cache != nil {
		w.cache.Delete(key)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(key))
	resp, _, _, err := w.request(ctx, http.MethodDelete, path, nil, make(http.Header))
	if err != nil {
		return err
	}

	a := &apiResponse{}
	if err := json.Unmarshal(resp, a); err != nil {
		return err
	}

	if !a.Success {
		messages := ""
		for _, m := range a.Errors {
			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
		}
		return errors.New(messages)
	}

	return nil
}

func (w *workersKV) request(ctx context.Context, method, path string, body interface{}, headers http.Header) ([]byte, http.Header, int, error) {
	var jsonBody []byte
	var err error

	if body != nil {
		if paramBytes, ok := body.([]byte); ok {
			jsonBody = paramBytes
		} else {
			jsonBody, err = json.Marshal(body)
			if err != nil {
				return nil, nil, 0, errors.Wrap(err, "error marshalling params to JSON")
			}
		}
	} else {
		jsonBody = nil
	}

	var reqBody io.Reader

	if jsonBody != nil {
		reqBody = bytes.NewReader(jsonBody)
	}

	req, err := http.NewRequestWithContext(ctx, method, apiBaseURL+path, reqBody)
	if err != nil {
		return nil, nil, 0, errors.Wrap(err, "error creating new request")
	}

	for key, value := range headers {
		req.Header[key] = value
	}

	// set token if it exists
	if len(w.token) > 0 {
		req.Header.Set("Authorization", "Bearer "+w.token)
	}

	// set the user agent to micro
	req.Header.Set("User-Agent", "micro/1.0 (https://micro.mu)")

	// Official cloudflare client does exponential backoff here
	// TODO: retry and use util/backoff
	resp, err := w.httpClient.Do(req)
	if err != nil {
		return nil, nil, 0, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return respBody, resp.Header, resp.StatusCode, err
	}

	return respBody, resp.Header, resp.StatusCode, nil
}

func (w *workersKV) String() string {
	return "cloudflare"
}

// NewStore returns a cloudflare Store implementation.
// Account ID, Token and Namespace must either be passed as options or
// environment variables. If set as env vars we expect the following;
// CF_API_TOKEN to a cloudflare API token scoped to Workers KV.
// CF_ACCOUNT_ID to contain a string with your cloudflare account ID.
// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage.
func NewStore(opts ...store.Option) store.Store {
	var options store.Options
	for _, o := range opts {
		o(&options)
	}

	// get options from environment
	account, token, namespace := getOptions()

	if len(account) == 0 {
		account = getAccount(options.Context)
	}

	if len(token) == 0 {
		token = getToken(options.Context)
	}

	if len(namespace) == 0 {
		namespace = options.Namespace
	}

	// validate options are not blank or log.Fatal
	validateOptions(account, token, namespace)

	return &workersKV{
		account:    account,
		namespace:  namespace,
		token:      token,
		options:    options,
		httpClient: &http.Client{},
	}
}
<|endoftext|>


<commit_before>package ddl

import (
	"database/sql"
	"errors"

	"github.com/drone/drone/store/datastore/ddl/mysql"
	"github.com/drone/drone/store/datastore/ddl/postgres"
	"github.com/drone/drone/store/datastore/ddl/sqlite"
)

// Supported database drivers
const (
	DriverSqlite   = "sqlite3"
	DriverMysql    = "mysql"
	DriverPostgres = "postgres"
)

// Migrate performs the database migration. If the migration fails
// and error is returned.
func Migrate(driver string, db *sql.DB) error {
	switch driver {
	case DriverMysql:
		if err := checkPriorMigration(db); err != nil {
			return err
		}
		return mysql.Migrate(db)
	case DriverPostgres:
		return postgres.Migrate(db)
	default:
		return sqlite.Migrate(db)
	}
}

// we need to check and see if there was a previous migration
// for drone 0.6 or prior and migrate to the new migration
// system. Attempting to migrate from 0.5 or below to 0.7 or
// above will result in an error.
//
// this can be removed once we get to 1.0 with the reasonable
// expectation that people are no longer using 0.5.
func checkPriorMigration(db *sql.DB) error {
	var none int
	if err := db.QueryRow(legacyMigrationsExist).Scan(&none); err != nil {
		// if no legacy migrations exist, this is a fresh install
		// and we can proceed as normal.
		return nil
	}
	if err := db.QueryRow(legacyMigrationsCurrent).Scan(&none); err != nil {
		// this indicates an attempted upgrade from 0.5 or lower to
		// version 0.7 or higher and will fail.
		return errors.New("Please upgrade to 0.6 before upgrading to 0.7+")
	}
	if _, err := db.Exec(createMigrationsTable); err != nil {
		return err
	}
	if _, err := db.Exec(legacyMigrationsImport); err != nil {
		return err
	}
	return nil
}

var legacyMigrationsExist = `
SELECT 1
FROM gorp_migrations
LIMIT 1
`

var legacyMigrationsCurrent = `
SELECT 1
FROM gorp_migrations
WHERE id = '16.sql'
LIMIT 1
`

var legacyMigrationsImport = `
INSERT OR IGNORE INTO migrations (name) VALUES
 ('create-table-users')
,('create-table-repos')
,('create-table-builds')
,('create-index-builds-repo')
,('create-index-builds-author')
,('create-table-procs')
,('create-index-procs-build')
,('create-table-logs')
,('create-table-files')
,('create-index-files-builds')
,('create-index-files-procs')
,('create-table-secrets')
,('create-index-secrets-repo')
,('create-table-registry')
,('create-index-registry-repo')
,('create-table-config')
,('create-table-tasks')
,('create-table-agents')
,('create-table-senders')
,('create-index-sender-repos')
`

var createMigrationsTable = `
CREATE TABLE IF NOT EXISTS migrations (
 name VARCHAR(512)
,UNIQUE(name)
)
`
<commit_msg>fix insert ignore syntax for mysql<commit_after>package ddl

import (
	"database/sql"
	"errors"

	"github.com/drone/drone/store/datastore/ddl/mysql"
	"github.com/drone/drone/store/datastore/ddl/postgres"
	"github.com/drone/drone/store/datastore/ddl/sqlite"
)

// Supported database drivers
const (
	DriverSqlite   = "sqlite3"
	DriverMysql    = "mysql"
	DriverPostgres = "postgres"
)

// Migrate performs the database migration. If the migration fails
// and error is returned.
func Migrate(driver string, db *sql.DB) error {
	switch driver {
	case DriverMysql:
		if err := checkPriorMigration(db); err != nil {
			return err
		}
		return mysql.Migrate(db)
	case DriverPostgres:
		return postgres.Migrate(db)
	default:
		return sqlite.Migrate(db)
	}
}

// we need to check and see if there was a previous migration
// for drone 0.6 or prior and migrate to the new migration
// system. Attempting to migrate from 0.5 or below to 0.7 or
// above will result in an error.
//
// this can be removed once we get to 1.0 with the reasonable
// expectation that people are no longer using 0.5.
func checkPriorMigration(db *sql.DB) error {
	var none int
	if err := db.QueryRow(legacyMigrationsExist).Scan(&none); err != nil {
		// if no legacy migrations exist, this is a fresh install
		// and we can proceed as normal.
		return nil
	}
	if err := db.QueryRow(legacyMigrationsCurrent).Scan(&none); err != nil {
		// this indicates an attempted upgrade from 0.5 or lower to
		// version 0.7 or higher and will fail.
		return errors.New("Please upgrade to 0.6 before upgrading to 0.7+")
	}
	if _, err := db.Exec(createMigrationsTable); err != nil {
		return err
	}
	if _, err := db.Exec(legacyMigrationsImport); err != nil {
		return err
	}
	return nil
}

var legacyMigrationsExist = `
SELECT 1
FROM gorp_migrations
LIMIT 1
`

var legacyMigrationsCurrent = `
SELECT 1
FROM gorp_migrations
WHERE id = '16.sql'
LIMIT 1
`

var legacyMigrationsImport = `
INSERT IGNORE INTO migrations (name) VALUES
 ('create-table-users')
,('create-table-repos')
,('create-table-builds')
,('create-index-builds-repo')
,('create-index-builds-author')
,('create-table-procs')
,('create-index-procs-build')
,('create-table-logs')
,('create-table-files')
,('create-index-files-builds')
,('create-index-files-procs')
,('create-table-secrets')
,('create-index-secrets-repo')
,('create-table-registry')
,('create-index-registry-repo')
,('create-table-config')
,('create-table-tasks')
,('create-table-agents')
,('create-table-senders')
,('create-index-sender-repos')
`

var createMigrationsTable = `
CREATE TABLE IF NOT EXISTS migrations (
 name VARCHAR(512)
,UNIQUE(name)
)
`
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package githubUtil

import (
	"os"
	"testing"
)

func TestFilePathProfileToGithub(t *testing.T) {
	type args struct {
		file     string
		gopath   string
		repoRoot string
	}
	tests := []struct {
		name string
		args args
		want string
	}{
		{"repo on github.com",
			args{"github.com/myRepoOwner/myRepoName/pkg/ab/cde",
				"/d1/d2/d3/gopath",
				"/d1/d2/d3/gopath/src/github.com/myRepoOwner/myRepoName"},
			"pkg/ab/cde"},
		{"repo on knative.dev",
			args{"knative.dev/test-infra/pkg/ab/cde",
				"/d1/d2/gopath",
				"/d1/d2/gopath/src/knative.dev/test-infra"},
			"pkg/ab/cde"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gopath := os.Getenv("GOPATH")
			os.Setenv("GOPATH", tt.args.gopath)
			getRepoRootSaved := getRepoRoot
			getRepoRoot = func() (string, error) {
				return tt.args.repoRoot, nil
			}
			defer func(){
			    os.Setenv("GOPATH", gopath)
			    getRepoRoot = getRepoRootSaved
			}()
			if got := FilePathProfileToGithub(tt.args.file); got != tt.want {
				t.Errorf("FilePathProfileToGithub(%v) = %v, want %v", tt.args.file, got, tt.want)
			}
		})
	}
}
<commit_msg>golang format tools (#1347)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package githubUtil

import (
	"os"
	"testing"
)

func TestFilePathProfileToGithub(t *testing.T) {
	type args struct {
		file     string
		gopath   string
		repoRoot string
	}
	tests := []struct {
		name string
		args args
		want string
	}{
		{"repo on github.com",
			args{"github.com/myRepoOwner/myRepoName/pkg/ab/cde",
				"/d1/d2/d3/gopath",
				"/d1/d2/d3/gopath/src/github.com/myRepoOwner/myRepoName"},
			"pkg/ab/cde"},
		{"repo on knative.dev",
			args{"knative.dev/test-infra/pkg/ab/cde",
				"/d1/d2/gopath",
				"/d1/d2/gopath/src/knative.dev/test-infra"},
			"pkg/ab/cde"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gopath := os.Getenv("GOPATH")
			os.Setenv("GOPATH", tt.args.gopath)
			getRepoRootSaved := getRepoRoot
			getRepoRoot = func() (string, error) {
				return tt.args.repoRoot, nil
			}
			defer func() {
				os.Setenv("GOPATH", gopath)
				getRepoRoot = getRepoRootSaved
			}()
			if got := FilePathProfileToGithub(tt.args.file); got != tt.want {
				t.Errorf("FilePathProfileToGithub(%v) = %v, want %v", tt.args.file, got, tt.want)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package submission

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	"testing"
	"time"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/ctpolicy"
	"github.com/google/certificate-transparency-go/loglist"
	"github.com/google/certificate-transparency-go/testdata"
	"github.com/google/certificate-transparency-go/tls"
	"github.com/google/certificate-transparency-go/x509"
	"github.com/google/certificate-transparency-go/x509util"

	"github.com/google/go-cmp/cmp"
)

// readCertFile returns the first certificate it finds in file provided.
func readCertFile(filename string) string {
	data, err := x509util.ReadPossiblePEMFile(filename, "CERTIFICATE")
	if err != nil {
		return ""
	}
	return string(data[0])
}

type rootInfo struct {
	raw      string
	filename string
}

var (
	RootsCerts = map[string][]rootInfo{
		"ct.googleapis.com/aviator/": {
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
		},
		"ct.googleapis.com/rocketeer/": {
			rootInfo{filename: "../trillian/testdata/fake-ca.cert"},
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
			rootInfo{filename: "testdata/another.cert"},
		},
		"ct.googleapis.com/icarus/": {
			rootInfo{raw: "aW52YWxpZDAwMA=="}, // encoded 'invalid000'
			rootInfo{filename: "testdata/another.cert"},
		},
		"uncollectable-roots/log/": {
			rootInfo{raw: "invalid"},
		},
	}
)

// buildNoLogClient is LogClientBuilder that always fails.
func buildNoLogClient(_ *loglist.Log) (client.AddLogClient, error) {
	return nil, errors.New("bad client builder")
}

// Stub for AddLogClient interface
type emptyLogClient struct {
}

func (e emptyLogClient) AddChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return nil, nil
}

func (e emptyLogClient) AddPreChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return nil, nil
}

func (e emptyLogClient) GetAcceptedRoots(ctx context.Context) ([]ct.ASN1Cert, error) {
	return nil, nil
}

// buildEmptyLogClient produces empty stub Log clients.
func buildEmptyLogClient(_ *loglist.Log) (client.AddLogClient, error) {
	return emptyLogClient{}, nil
}

func sampleLogList(t *testing.T) *loglist.LogList {
	t.Helper()
	var loglist loglist.LogList
	err := json.Unmarshal([]byte(testdata.SampleLogList), &loglist)
	if err != nil {
		t.Fatalf("Unable to Unmarshal testdata.SampleLogList %v", err)
	}
	return &loglist
}

func sampleValidLogList(t *testing.T) *loglist.LogList {
	t.Helper()
	ll := sampleLogList(t)
	// Id of invalid Log description Racketeer
	inval := 3
	ll.Logs = append(ll.Logs[:inval], ll.Logs[inval+1:]...)
	return ll
}

func sampleUncollectableLogList(t *testing.T) *loglist.LogList {
	t.Helper()
	ll := sampleValidLogList(t)
	// Append loglist that is unable to provide roots on request.
	ll.Logs = append(ll.Logs, loglist.Log{
		Description: "Does not return roots", Key: []byte("VW5jb2xsZWN0YWJsZUxvZ0xpc3Q="),
		MaximumMergeDelay: 123, OperatedBy: []int{0},
		URL:            "uncollectable-roots/log/",
		DNSAPIEndpoint: "uncollectavle.ct.googleapis.com",
	})
	return ll
}

func TestNewDistributorLogClients(t *testing.T) {
	testCases := []struct {
		name      string
		ll        *loglist.LogList
		lcBuilder LogClientBuilder
		errRegexp *regexp.Regexp
	}{
		{
			name:      "ValidLogClients",
			ll:        sampleValidLogList(t),
			lcBuilder: buildEmptyLogClient,
		},
		{
			name:      "NoLogClients",
			ll:        sampleValidLogList(t),
			lcBuilder: buildNoLogClient,
			errRegexp: regexp.MustCompile("failed to create log client"),
		},
		{
			name:      "NoLogClientsEmptyLogList",
			ll:        &loglist.LogList{},
			lcBuilder: buildNoLogClient,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			_, err := NewDistributor(tc.ll, ctpolicy.ChromeCTPolicy{}, tc.lcBuilder)
			if gotErr, wantErr := err != nil, tc.errRegexp != nil; gotErr != wantErr {
				var unwantedErr string
				if gotErr {
					unwantedErr = fmt.Sprintf(" (%q)", err)
				}
				t.Errorf("Got error = %v%s, expected error = %v", gotErr, unwantedErr, wantErr)
			} else if tc.errRegexp != nil && !tc.errRegexp.MatchString(err.Error()) {
				t.Errorf("Error %q did not match expected regexp %q", err, tc.errRegexp)
			}
		})
	}
}

// TestSCT builds a mock SCT for given logURL.
func testSCT(logURL string) *ct.SignedCertificateTimestamp {
	var keyID [sha256.Size]byte
	copy(keyID[:], logURL)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		LogID:      ct.LogID{KeyID: keyID},
		Timestamp:  1234,
		Extensions: []byte{},
		Signature: ct.DigitallySigned{
			Algorithm: tls.SignatureAndHashAlgorithm{
				Hash:      tls.SHA256,
				Signature: tls.ECDSA,
			},
		},
	}
}

// Stub for AddLogCLient interface
type stubLogClient struct {
	logURL string
}

func (m stubLogClient) AddChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return nil, nil
}

func (m stubLogClient) AddPreChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	if _, ok := RootsCerts[m.logURL]; ok {
		return testSCT(m.logURL), nil
	}
	return nil, fmt.Errorf("log %q has no roots", m.logURL)
}

func (m stubLogClient) GetAcceptedRoots(ctx context.Context) ([]ct.ASN1Cert, error) {
	roots := []ct.ASN1Cert{}
	if certInfos, ok := RootsCerts[m.logURL]; ok {
		for _, certInfo := range certInfos {
			if len(certInfo.raw) > 0 {
				roots = append(roots, ct.ASN1Cert{Data: []byte(certInfo.raw)})
			} else {

				roots = append(roots, ct.ASN1Cert{Data: []byte(readCertFile(certInfo.filename))})
			}
		}
	}
	return roots, nil
}

func buildStubLogClient(log *loglist.Log) (client.AddLogClient, error) {
	return stubLogClient{logURL: log.URL}, nil
}

func TestNewDistributorRootPools(t *testing.T) {
	testCases := []struct {
		name    string
		ll      *loglist.LogList
		rootNum map[string]int
	}{
		{
			name:    "InactiveZeroRoots",
			ll:      sampleValidLogList(t),
			rootNum: map[string]int{"ct.googleapis.com/aviator/": 0, "ct.googleapis.com/rocketeer/": 4, "ct.googleapis.com/icarus/": 1}, // aviator is not active; 1 of 2 icarus roots is not x509 struct
		},
		{
			name:    "CouldNotCollect",
			ll:      sampleUncollectableLogList(t),
			rootNum: map[string]int{"ct.googleapis.com/aviator/": 0, "ct.googleapis.com/rocketeer/": 4, "ct.googleapis.com/icarus/": 1, "uncollectable-roots/log/": 0}, // aviator is not active; uncollectable client cannot provide roots
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			dist, _ := NewDistributor(tc.ll, ctpolicy.ChromeCTPolicy{}, buildStubLogClient)
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			go dist.Run(ctx)
			// First Log refresh expected.
			<-ctx.Done()

			dist.mu.Lock()
			defer dist.mu.Unlock()
			for logURL, wantNum := range tc.rootNum {
				gotNum := 0
				if roots, ok := dist.logRoots[logURL]; ok {
					gotNum = len(roots.RawCertificates())
				}
				if wantNum != gotNum {
					t.Errorf("Expected %d root(s) for Log %s, got %d", wantNum, logURL, gotNum)
				}
			}
		})
	}
}

func pemFileToDERChain(t *testing.T, filename string) [][]byte {
	t.Helper()
	rawChain, err := x509util.ReadPossiblePEMFile(filename, "CERTIFICATE")
	if err != nil {
		t.Fatalf("failed to load testdata: %v", err)
	}
	return rawChain
}

func getSCTMap(l []*AssignedSCT) map[string]*AssignedSCT {
	m := map[string]*AssignedSCT{}
	for _, asct := range l {
		m[asct.LogURL] = asct
	}
	return m
}

// Stub CT policy to run tests.
type stubCTPolicy struct {
	baseNum int
}

// Builds simplistic policy requiring n SCTs from any Logs for each cert.
func buildStubCTPolicy(n int) stubCTPolicy {
	return stubCTPolicy{baseNum: n}
}

func (stubP stubCTPolicy) LogsByGroup(cert *x509.Certificate, approved *loglist.LogList) (ctpolicy.LogPolicyData, error) {
	baseGroup, err := ctpolicy.BaseGroupFor(approved, stubP.baseNum)
	groups := ctpolicy.LogPolicyData{baseGroup.Name: &baseGroup}
	return groups, err
}

func TestDistributorAddPreChain(t *testing.T) {
	testCases := []struct {
		name     string
		ll       *loglist.LogList
		plc      ctpolicy.CTPolicy
		rawChain [][]byte
		scts     []*AssignedSCT
		wantErr  bool
	}{
		{
			name:     "MalformedChainRequest",
			ll:       sampleValidLogList(t),
			plc:      ctpolicy.ChromeCTPolicy{},
			rawChain: pemFileToDERChain(t, "../trillian/testdata/subleaf.misordered.chain"),
			scts:     nil,
			wantErr:  true,
		},
		{
			name:     "CallBeforeInit",
			ll:       sampleValidLogList(t),
			plc:      ctpolicy.ChromeCTPolicy{},
			rawChain: nil,
			scts:     nil,
			wantErr:  true,
		},
		{
			name:     "InsufficientSCTsForPolicy",
			ll:       sampleValidLogList(t),
			plc:      ctpolicy.AppleCTPolicy{},
			rawChain: pemFileToDERChain(t, "../trillian/testdata/subleaf.chain"), // subleaf chain is fake-ca-1-rooted
			scts:     []*AssignedSCT{},
			wantErr:  true, // Not enough SCTs for policy
		},
		{
			name:     "FullChain1Policy",
			ll:       sampleValidLogList(t),
			plc:      buildStubCTPolicy(1),
			rawChain: pemFileToDERChain(t, "../trillian/testdata/subleaf.chain"),
			scts: []*AssignedSCT{
				{
					LogURL: "ct.googleapis.com/rocketeer/",
					SCT:    testSCT("ct.googleapis.com/rocketeer/"),
				},
			},
			wantErr: false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			dist, _ := NewDistributor(tc.ll, tc.plc, buildStubLogClient)
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			dist.Run(ctx)

			scts, err := dist.AddPreChain(context.Background(), tc.rawChain)
			if gotErr := (err != nil); gotErr != tc.wantErr {
				t.Errorf("Expected to get errors is %v while actually getting errors is %v", tc.wantErr, gotErr)
			}

			if got, want := len(scts), len(tc.scts); got != want {
				t.Errorf("Expected to get %d SCTs on AddPreChain request, got %d", want, got)
			}
			gotMap := getSCTMap(tc.scts)
			for _, asct := range scts {
				if wantedSCT, ok := gotMap[asct.LogURL]; !ok {
					t.Errorf("dist.AddPreChain() = (_, %v), want err? %t", err, tc.wantErr)
				} else if diff := cmp.Diff(asct, wantedSCT); diff != "" {
					t.Errorf("Got unexpected SCT for Log %q", asct.LogURL)
				}
			}
		})
	}
}
<commit_msg>Return []byte from readCertFile() instead of string<commit_after>// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package submission

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	"testing"
	"time"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/ctpolicy"
	"github.com/google/certificate-transparency-go/loglist"
	"github.com/google/certificate-transparency-go/testdata"
	"github.com/google/certificate-transparency-go/tls"
	"github.com/google/certificate-transparency-go/x509"
	"github.com/google/certificate-transparency-go/x509util"

	"github.com/google/go-cmp/cmp"
)

// readCertFile returns the first certificate it finds in file provided.
func readCertFile(filename string) []byte {
	data, err := x509util.ReadPossiblePEMFile(filename, "CERTIFICATE")
	if err != nil {
		return nil
	}
	return data[0]
}

type rootInfo struct {
	raw      string
	filename string
}

var (
	RootsCerts = map[string][]rootInfo{
		"ct.googleapis.com/aviator/": {
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
		},
		"ct.googleapis.com/rocketeer/": {
			rootInfo{filename: "../trillian/testdata/fake-ca.cert"},
			rootInfo{filename: "../trillian/testdata/fake-ca-1.cert"},
			rootInfo{filename: "testdata/some.cert"},
			rootInfo{filename: "testdata/another.cert"},
		},
		"ct.googleapis.com/icarus/": {
			rootInfo{raw: "aW52YWxpZDAwMA=="}, // encoded 'invalid000'
			rootInfo{filename: "testdata/another.cert"},
		},
		"uncollectable-roots/log/": {
			rootInfo{raw: "invalid"},
		},
	}
)

// buildNoLogClient is LogClientBuilder that always fails.
func buildNoLogClient(_ *loglist.Log) (client.AddLogClient, error) {
	return nil, errors.New("bad client builder")
}

// Stub for AddLogClient interface
type emptyLogClient struct {
}

func (e emptyLogClient) AddChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return nil, nil
}

func (e emptyLogClient) AddPreChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return nil, nil
}

func (e emptyLogClient) GetAcceptedRoots(ctx context.Context) ([]ct.ASN1Cert, error) {
	return nil, nil
}

// buildEmptyLogClient produces empty stub Log clients.
func buildEmptyLogClient(_ *loglist.Log) (client.AddLogClient, error) {
	return emptyLogClient{}, nil
}

func sampleLogList(t *testing.T) *loglist.LogList {
	t.Helper()
	var loglist loglist.LogList
	err := json.Unmarshal([]byte(testdata.SampleLogList), &loglist)
	if err != nil {
		t.Fatalf("Unable to Unmarshal testdata.SampleLogList %v", err)
	}
	return &loglist
}

func sampleValidLogList(t *testing.T) *loglist.LogList {
	t.Helper()
	ll := sampleLogList(t)
	// Id of invalid Log description Racketeer
	inval := 3
	ll.Logs = append(ll.Logs[:inval], ll.Logs[inval+1:]...)
	return ll
}

func sampleUncollectableLogList(t *testing.T) *loglist.LogList {
	t.Helper()
	ll := sampleValidLogList(t)
	// Append loglist that is unable to provide roots on request.
	ll.Logs = append(ll.Logs, loglist.Log{
		Description: "Does not return roots", Key: []byte("VW5jb2xsZWN0YWJsZUxvZ0xpc3Q="),
		MaximumMergeDelay: 123, OperatedBy: []int{0},
		URL:            "uncollectable-roots/log/",
		DNSAPIEndpoint: "uncollectavle.ct.googleapis.com",
	})
	return ll
}

func TestNewDistributorLogClients(t *testing.T) {
	testCases := []struct {
		name      string
		ll        *loglist.LogList
		lcBuilder LogClientBuilder
		errRegexp *regexp.Regexp
	}{
		{
			name:      "ValidLogClients",
			ll:        sampleValidLogList(t),
			lcBuilder: buildEmptyLogClient,
		},
		{
			name:      "NoLogClients",
			ll:        sampleValidLogList(t),
			lcBuilder: buildNoLogClient,
			errRegexp: regexp.MustCompile("failed to create log client"),
		},
		{
			name:      "NoLogClientsEmptyLogList",
			ll:        &loglist.LogList{},
			lcBuilder: buildNoLogClient,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			_, err := NewDistributor(tc.ll, ctpolicy.ChromeCTPolicy{}, tc.lcBuilder)
			if gotErr, wantErr := err != nil, tc.errRegexp != nil; gotErr != wantErr {
				var unwantedErr string
				if gotErr {
					unwantedErr = fmt.Sprintf(" (%q)", err)
				}
				t.Errorf("Got error = %v%s, expected error = %v", gotErr, unwantedErr, wantErr)
			} else if tc.errRegexp != nil && !tc.errRegexp.MatchString(err.Error()) {
				t.Errorf("Error %q did not match expected regexp %q", err, tc.errRegexp)
			}
		})
	}
}

// TestSCT builds a mock SCT for given logURL.
func testSCT(logURL string) *ct.SignedCertificateTimestamp {
	var keyID [sha256.Size]byte
	copy(keyID[:], logURL)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		LogID:      ct.LogID{KeyID: keyID},
		Timestamp:  1234,
		Extensions: []byte{},
		Signature: ct.DigitallySigned{
			Algorithm: tls.SignatureAndHashAlgorithm{
				Hash:      tls.SHA256,
				Signature: tls.ECDSA,
			},
		},
	}
}

// Stub for AddLogCLient interface
type stubLogClient struct {
	logURL string
}

func (m stubLogClient) AddChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return nil, nil
}

func (m stubLogClient) AddPreChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	if _, ok := RootsCerts[m.logURL]; ok {
		return testSCT(m.logURL), nil
	}
	return nil, fmt.Errorf("log %q has no roots", m.logURL)
}

func (m stubLogClient) GetAcceptedRoots(ctx context.Context) ([]ct.ASN1Cert, error) {
	roots := []ct.ASN1Cert{}
	if certInfos, ok := RootsCerts[m.logURL]; ok {
		for _, certInfo := range certInfos {
			if len(certInfo.raw) > 0 {
				roots = append(roots, ct.ASN1Cert{Data: []byte(certInfo.raw)})
			} else {

				roots = append(roots, ct.ASN1Cert{Data: readCertFile(certInfo.filename)})
			}
		}
	}
	return roots, nil
}

func buildStubLogClient(log *loglist.Log) (client.AddLogClient, error) {
	return stubLogClient{logURL: log.URL}, nil
}

func TestNewDistributorRootPools(t *testing.T) {
	testCases := []struct {
		name    string
		ll      *loglist.LogList
		rootNum map[string]int
	}{
		{
			name:    "InactiveZeroRoots",
			ll:      sampleValidLogList(t),
			rootNum: map[string]int{"ct.googleapis.com/aviator/": 0, "ct.googleapis.com/rocketeer/": 4, "ct.googleapis.com/icarus/": 1}, // aviator is not active; 1 of 2 icarus roots is not x509 struct
		},
		{
			name:    "CouldNotCollect",
			ll:      sampleUncollectableLogList(t),
			rootNum: map[string]int{"ct.googleapis.com/aviator/": 0, "ct.googleapis.com/rocketeer/": 4, "ct.googleapis.com/icarus/": 1, "uncollectable-roots/log/": 0}, // aviator is not active; uncollectable client cannot provide roots
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			dist, _ := NewDistributor(tc.ll, ctpolicy.ChromeCTPolicy{}, buildStubLogClient)
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			go dist.Run(ctx)
			// First Log refresh expected.
			<-ctx.Done()

			dist.mu.Lock()
			defer dist.mu.Unlock()
			for logURL, wantNum := range tc.rootNum {
				gotNum := 0
				if roots, ok := dist.logRoots[logURL]; ok {
					gotNum = len(roots.RawCertificates())
				}
				if wantNum != gotNum {
					t.Errorf("Expected %d root(s) for Log %s, got %d", wantNum, logURL, gotNum)
				}
			}
		})
	}
}

func pemFileToDERChain(t *testing.T, filename string) [][]byte {
	t.Helper()
	rawChain, err := x509util.ReadPossiblePEMFile(filename, "CERTIFICATE")
	if err != nil {
		t.Fatalf("failed to load testdata: %v", err)
	}
	return rawChain
}

func getSCTMap(l []*AssignedSCT) map[string]*AssignedSCT {
	m := map[string]*AssignedSCT{}
	for _, asct := range l {
		m[asct.LogURL] = asct
	}
	return m
}

// Stub CT policy to run tests.
type stubCTPolicy struct {
	baseNum int
}

// Builds simplistic policy requiring n SCTs from any Logs for each cert.
func buildStubCTPolicy(n int) stubCTPolicy {
	return stubCTPolicy{baseNum: n}
}

func (stubP stubCTPolicy) LogsByGroup(cert *x509.Certificate, approved *loglist.LogList) (ctpolicy.LogPolicyData, error) {
	baseGroup, err := ctpolicy.BaseGroupFor(approved, stubP.baseNum)
	groups := ctpolicy.LogPolicyData{baseGroup.Name: &baseGroup}
	return groups, err
}

func TestDistributorAddPreChain(t *testing.T) {
	testCases := []struct {
		name     string
		ll       *loglist.LogList
		plc      ctpolicy.CTPolicy
		rawChain [][]byte
		scts     []*AssignedSCT
		wantErr  bool
	}{
		{
			name:     "MalformedChainRequest",
			ll:       sampleValidLogList(t),
			plc:      ctpolicy.ChromeCTPolicy{},
			rawChain: pemFileToDERChain(t, "../trillian/testdata/subleaf.misordered.chain"),
			scts:     nil,
			wantErr:  true,
		},
		{
			name:     "CallBeforeInit",
			ll:       sampleValidLogList(t),
			plc:      ctpolicy.ChromeCTPolicy{},
			rawChain: nil,
			scts:     nil,
			wantErr:  true,
		},
		{
			name:     "InsufficientSCTsForPolicy",
			ll:       sampleValidLogList(t),
			plc:      ctpolicy.AppleCTPolicy{},
			rawChain: pemFileToDERChain(t, "../trillian/testdata/subleaf.chain"), // subleaf chain is fake-ca-1-rooted
			scts:     []*AssignedSCT{},
			wantErr:  true, // Not enough SCTs for policy
		},
		{
			name:     "FullChain1Policy",
			ll:       sampleValidLogList(t),
			plc:      buildStubCTPolicy(1),
			rawChain: pemFileToDERChain(t, "../trillian/testdata/subleaf.chain"),
			scts: []*AssignedSCT{
				{
					LogURL: "ct.googleapis.com/rocketeer/",
					SCT:    testSCT("ct.googleapis.com/rocketeer/"),
				},
			},
			wantErr: false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			dist, _ := NewDistributor(tc.ll, tc.plc, buildStubLogClient)
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			dist.Run(ctx)

			scts, err := dist.AddPreChain(context.Background(), tc.rawChain)
			if gotErr := (err != nil); gotErr != tc.wantErr {
				t.Errorf("Expected to get errors is %v while actually getting errors is %v", tc.wantErr, gotErr)
			}

			if got, want := len(scts), len(tc.scts); got != want {
				t.Errorf("Expected to get %d SCTs on AddPreChain request, got %d", want, got)
			}
			gotMap := getSCTMap(tc.scts)
			for _, asct := range scts {
				if wantedSCT, ok := gotMap[asct.LogURL]; !ok {
					t.Errorf("dist.AddPreChain() = (_, %v), want err? %t", err, tc.wantErr)
				} else if diff := cmp.Diff(asct, wantedSCT); diff != "" {
					t.Errorf("Got unexpected SCT for Log %q", asct.LogURL)
				}
			}
		})
	}
}
<|endoftext|>