<commit_before>package syntax

import (
	"regexp"
	"unicode/utf8"

	termbox "github.com/nsf/termbox-go"
)

func init() {
	Languages["go"] = Language{
		Syntax{"string", regexp.MustCompile(`^(?m)".*?(?:[^\\]?"|$)`), termbox.ColorRed, termbox.ColorBlack},
		Syntax{"raw string", regexp.MustCompile(`^(?s)` + "`" + `.*?` + "(?:`|$)"), termbox.ColorRed, termbox.ColorBlack},
		Syntax{"rune", regexp.MustCompile(`^(?m)'.*?(?:[^\\]?'|$)`), termbox.ColorYellow, termbox.ColorBlack},
		Syntax{"comment", regexp.MustCompile(`^(?m)//.*`), termbox.ColorMagenta, termbox.ColorBlack},
		Syntax{"multi line comment", regexp.MustCompile(`^(?s)/[*].*?(?:[*]/|$)`), termbox.ColorMagenta, termbox.ColorBlack},
		Syntax{"trailing spaces", regexp.MustCompile(`^(?m)[ \t]+$`), termbox.ColorBlack, termbox.ColorYellow},
		Syntax{"package", regexp.MustCompile(`^package\s`), termbox.ColorYellow, termbox.ColorBlack},
	}

	Languages["py"] = Language{
		Syntax{"multi line string1", regexp.MustCompile(`^(?s)""".*?(?:"""|$)`), termbox.ColorRed, termbox.ColorBlack},
		Syntax{"multi line string2", regexp.MustCompile(`^(?s)'''.*?(?:'''|$)`), termbox.ColorYellow, termbox.ColorBlack},
		Syntax{"string1", regexp.MustCompile(`^(?m)".*?(?:[^\\]?"|$)`), termbox.ColorRed, termbox.ColorBlack},
		Syntax{"string2", regexp.MustCompile(`^(?m)'.*?(?:[^\\]?'|$)`), termbox.ColorYellow, termbox.ColorBlack},
		Syntax{"comment", regexp.MustCompile(`^(?m)#.*`), termbox.ColorMagenta, termbox.ColorBlack},
		Syntax{"trailing spaces", regexp.MustCompile(`^(?m)[ \t]+$`), termbox.ColorBlack, termbox.ColorYellow},
	}
}

type Syntax struct {
	Name string
	Re   *regexp.Regexp
	Fg   termbox.Attribute
	Bg   termbox.Attribute
}

func (s Syntax) NewMatch(start, end Pos) Match {
	return Match{Name: s.Name, Start: start, End: end, Fg: s.Fg, Bg: s.Bg}
}

type Language []Syntax

var Languages = make(map[string]Language)

func (l Language) Parse(text []byte) []Match {
	c := NewCursor(text)
	matches := []Match{}
Loop:
	for {
		for _, syn := range l {
			ms := syn.Re.FindSubmatch(c.Remain())
			if ms == nil {
				continue
			}
			m := ms[0]
			if len(ms) == 2 {
				m = ms[1]
			}
			if string(m) == "" {
				continue
			}
			start := c.Pos()
			c.Skip(len(m))
			end := c.Pos()
			matches = append(matches, syn.NewMatch(start, end))
			continue Loop
		}
		if !c.Advance() {
			break
		}
	}
	return matches
}

// ParseRange checks and replace matches from min to max.
// When there is an overwrap between old matches and min Pos,
// it will recaculate matches from the overwrap begins.
func (l Language) ParseRange(matches []Match, text []byte, min, max Pos) []Match {
	// check where parse acutally started.
	last := -1
	overwrap := false
	for i, m := range matches {
		if m.Min().Compare(min) < 0 {
			if m.Max().Compare(min) < 0 {
				continue
			}
			overwrap = true
			last = i
			break
		}
		last = i
		break
	}

	parseStart := min
	if last != -1 {
		if overwrap {
			parseStart = matches[last].Min()
		}
		matches = matches[:last]
	}

	// move cursor to start position.
	c := NewCursor(text)
	for c.Pos().Compare(parseStart) < 0 {
		ok := c.Advance()
		if !ok {
			// already end of text. nothing to do.
			return matches
		}
	}

Loop:
	for {
		if c.Pos().Compare(max) >= 0 {
			break
		}
		for _, syn := range l {
			ms := syn.Re.FindSubmatch(c.Remain())
			if ms == nil {
				continue
			}
			m := ms[0]
			if len(ms) == 2 {
				m = ms[1]
			}
			if string(m) == "" {
				continue
			}
			start := c.Pos()
			c.Skip(len(m))
			end := c.Pos()
			matches = append(matches, syn.NewMatch(start, end))
			continue Loop
		}
		if !c.Advance() {
			break
		}
	}
	return matches
}

type Cursor struct {
	text []byte
	b    int // byte offset
	l    int // line offset
	o    int // byte in line offset
}

func NewCursor(text []byte) *Cursor {
	return &Cursor{text: text}
}

func (c *Cursor) Pos() Pos {
	return Pos{c.l, c.o}
}

func (c *Cursor) Remain() []byte {
	if c.l == len(c.text) {
		return []byte("")
	}
	return c.text[c.b:]
}

func (c *Cursor) Advance() bool {
	if c.b == len(c.text) {
		return false
	}
	c.next()
	return true
}

func (c *Cursor) Skip(b int) {
	i := 0
	for i < b {
		_, size := c.next()
		i += size
	}
}

func (c *Cursor) next() (r rune, size int) {
	r, size = utf8.DecodeRune(c.Remain())
	c.b += size
	c.o += size
	if r == '\n' {
		c.l += 1
		c.o = 0
	}
	return r, size
}

type Match struct {
	Name  string
	Start Pos
	End   Pos
	Fg    termbox.Attribute
	Bg    termbox.Attribute
}

func (m *Match) MinMax() (Pos, Pos) {
	if m.Start.L < m.End.L {
		return m.Start, m.End
	}
	if m.Start.L == m.End.L && m.Start.O <= m.End.O {
		return m.Start, m.End
	}
	return m.End, m.Start
}

func (m *Match) Min() Pos {
	min, _ := m.MinMax()
	return min
}

func (m *Match) Max() Pos {
	_, max := m.MinMax()
	return max
}

type Pos struct {
	L int
	O int
}

// Compare compares two Pos a and b.
// If a < b, it will return -1.
// If a > b, it will return 1.
// If a == b, it will return 0
func (a Pos) Compare(b Pos) int {
	if a.L < b.L {
		return -1
	}
	if a.L == b.L {
		if a.O < b.O {
			return -1
		}
		if a.O == b.O {
			return 0
		}
		return 1
	}
	return 1
}
<commit_msg>termbox to term for consistency<commit_after>package syntax

import (
	"regexp"
	"unicode/utf8"

	term "github.com/nsf/termbox-go"
)

func init() {
	Languages["go"] = Language{
		Syntax{"string", regexp.MustCompile(`^(?m)".*?(?:[^\\]?"|$)`), term.ColorRed, term.ColorBlack},
		Syntax{"raw string", regexp.MustCompile(`^(?s)` + "`" + `.*?` + "(?:`|$)"), term.ColorRed, term.ColorBlack},
		Syntax{"rune", regexp.MustCompile(`^(?m)'.*?(?:[^\\]?'|$)`), term.ColorYellow, term.ColorBlack},
		Syntax{"comment", regexp.MustCompile(`^(?m)//.*`), term.ColorMagenta, term.ColorBlack},
		Syntax{"multi line comment", regexp.MustCompile(`^(?s)/[*].*?(?:[*]/|$)`), term.ColorMagenta, term.ColorBlack},
		Syntax{"trailing spaces", regexp.MustCompile(`^(?m)[ \t]+$`), term.ColorBlack, term.ColorYellow},
		Syntax{"package", regexp.MustCompile(`^package\s`), term.ColorYellow, term.ColorBlack},
	}

	Languages["py"] = Language{
		Syntax{"multi line string1", regexp.MustCompile(`^(?s)""".*?(?:"""|$)`), term.ColorRed, term.ColorBlack},
		Syntax{"multi line string2", regexp.MustCompile(`^(?s)'''.*?(?:'''|$)`), term.ColorYellow, term.ColorBlack},
		Syntax{"string1", regexp.MustCompile(`^(?m)".*?(?:[^\\]?"|$)`), term.ColorRed, term.ColorBlack},
		Syntax{"string2", regexp.MustCompile(`^(?m)'.*?(?:[^\\]?'|$)`), term.ColorYellow, term.ColorBlack},
		Syntax{"comment", regexp.MustCompile(`^(?m)#.*`), term.ColorMagenta, term.ColorBlack},
		Syntax{"trailing spaces", regexp.MustCompile(`^(?m)[ \t]+$`), term.ColorBlack, term.ColorYellow},
	}
}

type Syntax struct {
	Name string
	Re   *regexp.Regexp
	Fg   term.Attribute
	Bg   term.Attribute
}

func (s Syntax) NewMatch(start, end Pos) Match {
	return Match{Name: s.Name, Start: start, End: end, Fg: s.Fg, Bg: s.Bg}
}

type Language []Syntax

var Languages = make(map[string]Language)

func (l Language) Parse(text []byte) []Match {
	c := NewCursor(text)
	matches := []Match{}
Loop:
	for {
		for _, syn := range l {
			ms := syn.Re.FindSubmatch(c.Remain())
			if ms == nil {
				continue
			}
			m := ms[0]
			if len(ms) == 2 {
				m = ms[1]
			}
			if string(m) == "" {
				continue
			}
			start := c.Pos()
			c.Skip(len(m))
			end := c.Pos()
			matches = append(matches, syn.NewMatch(start, end))
			continue Loop
		}
		if !c.Advance() {
			break
		}
	}
	return matches
}

// ParseRange checks and replace matches from min to max.
// When there is an overwrap between old matches and min Pos,
// it will recaculate matches from the overwrap begins.
func (l Language) ParseRange(matches []Match, text []byte, min, max Pos) []Match {
	// check where parse acutally started.
	last := -1
	overwrap := false
	for i, m := range matches {
		if m.Min().Compare(min) < 0 {
			if m.Max().Compare(min) < 0 {
				continue
			}
			overwrap = true
			last = i
			break
		}
		last = i
		break
	}

	parseStart := min
	if last != -1 {
		if overwrap {
			parseStart = matches[last].Min()
		}
		matches = matches[:last]
	}

	// move cursor to start position.
	c := NewCursor(text)
	for c.Pos().Compare(parseStart) < 0 {
		ok := c.Advance()
		if !ok {
			// already end of text. nothing to do.
			return matches
		}
	}

Loop:
	for {
		if c.Pos().Compare(max) >= 0 {
			break
		}
		for _, syn := range l {
			ms := syn.Re.FindSubmatch(c.Remain())
			if ms == nil {
				continue
			}
			m := ms[0]
			if len(ms) == 2 {
				m = ms[1]
			}
			if string(m) == "" {
				continue
			}
			start := c.Pos()
			c.Skip(len(m))
			end := c.Pos()
			matches = append(matches, syn.NewMatch(start, end))
			continue Loop
		}
		if !c.Advance() {
			break
		}
	}
	return matches
}

type Cursor struct {
	text []byte
	b    int // byte offset
	l    int // line offset
	o    int // byte in line offset
}

func NewCursor(text []byte) *Cursor {
	return &Cursor{text: text}
}

func (c *Cursor) Pos() Pos {
	return Pos{c.l, c.o}
}

func (c *Cursor) Remain() []byte {
	if c.l == len(c.text) {
		return []byte("")
	}
	return c.text[c.b:]
}

func (c *Cursor) Advance() bool {
	if c.b == len(c.text) {
		return false
	}
	c.next()
	return true
}

func (c *Cursor) Skip(b int) {
	i := 0
	for i < b {
		_, size := c.next()
		i += size
	}
}

func (c *Cursor) next() (r rune, size int) {
	r, size = utf8.DecodeRune(c.Remain())
	c.b += size
	c.o += size
	if r == '\n' {
		c.l += 1
		c.o = 0
	}
	return r, size
}

type Match struct {
	Name  string
	Start Pos
	End   Pos
	Fg    term.Attribute
	Bg    term.Attribute
}

func (m *Match) MinMax() (Pos, Pos) {
	if m.Start.L < m.End.L {
		return m.Start, m.End
	}
	if m.Start.L == m.End.L && m.Start.O <= m.End.O {
		return m.Start, m.End
	}
	return m.End, m.Start
}

func (m *Match) Min() Pos {
	min, _ := m.MinMax()
	return min
}

func (m *Match) Max() Pos {
	_, max := m.MinMax()
	return max
}

type Pos struct {
	L int
	O int
}

// Compare compares two Pos a and b.
// If a < b, it will return -1.
// If a > b, it will return 1.
// If a == b, it will return 0
func (a Pos) Compare(b Pos) int {
	if a.L < b.L {
		return -1
	}
	if a.L == b.L {
		if a.O < b.O {
			return -1
		}
		if a.O == b.O {
			return 0
		}
		return 1
	}
	return 1
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"io"
)

const DEFAULT_MAX_LENGTH = 16384000

type TFramedTransport struct {
	transport TTransport
	buf       bytes.Buffer
	reader    *bufio.Reader
	frameSize uint32 //Current remaining size of the frame. if ==0 read next frame header
	buffer    [4]byte
	maxLength uint32
}

type tFramedTransportFactory struct {
	factory   TTransportFactory
	maxLength uint32
}

func NewTFramedTransportFactory(factory TTransportFactory) TTransportFactory {
	return &tFramedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
}

func NewTFramedTransportFactoryMaxLength(factory TTransportFactory, maxLength uint32) TTransportFactory {
	return &tFramedTransportFactory{factory: factory, maxLength: maxLength}
}

func (p *tFramedTransportFactory) GetTransport(base TTransport) (TTransport, error) {
	tt, err := p.factory.GetTransport(base)
	if err != nil {
		return nil, err
	}
	return NewTFramedTransportMaxLength(tt, p.maxLength), nil
}

func NewTFramedTransport(transport TTransport) *TFramedTransport {
	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
}

func NewTFramedTransportMaxLength(transport TTransport, maxLength uint32) *TFramedTransport {
	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
}

func (p *TFramedTransport) Open() error {
	return p.transport.Open()
}

func (p *TFramedTransport) IsOpen() bool {
	return p.transport.IsOpen()
}

func (p *TFramedTransport) Close() error {
	return p.transport.Close()
}

func (p *TFramedTransport) Read(buf []byte) (l int, err error) {
	if p.frameSize == 0 {
		p.frameSize, err = p.readFrameHeader()
		if err != nil {
			return
		}
	}
	if p.frameSize < uint32(len(buf)) {
		frameSize := p.frameSize
		tmp := make([]byte, p.frameSize)
		l, err = p.Read(tmp)
		copy(buf, tmp)
		if err == nil {
			// Note: It's important to only return an error when l
			// is zero.
			// In io.Reader.Read interface, it's perfectly fine to
			// return partial data and nil error, which means
			// "This is all the data we have right now without
			// blocking. If you need the full data, call Read again
			// or use io.ReadFull instead".
			// Returning partial data with an error actually means
			// there's no more data after the partial data just
			// returned, which is not true in this case
			// (it might be that the other end just haven't written
			// them yet).
			if l == 0 {
				err = NewTTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", frameSize, len(buf)))
			}
			return
		}
	}
	got, err := p.reader.Read(buf)
	p.frameSize = p.frameSize - uint32(got)
	//sanity check
	if p.frameSize < 0 {
		return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "Negative frame size")
	}
	return got, NewTTransportExceptionFromError(err)
}

func (p *TFramedTransport) ReadByte() (c byte, err error) {
	if p.frameSize == 0 {
		p.frameSize, err = p.readFrameHeader()
		if err != nil {
			return
		}
	}
	if p.frameSize < 1 {
		return 0, NewTTransportExceptionFromError(fmt.Errorf("Not enough frame size %d to read %d bytes", p.frameSize, 1))
	}
	c, err = p.reader.ReadByte()
	if err == nil {
		p.frameSize--
	}
	return
}

func (p *TFramedTransport) Write(buf []byte) (int, error) {
	n, err := p.buf.Write(buf)
	return n, NewTTransportExceptionFromError(err)
}

func (p *TFramedTransport) WriteByte(c byte) error {
	return p.buf.WriteByte(c)
}

func (p *TFramedTransport) WriteString(s string) (n int, err error) {
	return p.buf.WriteString(s)
}

func (p *TFramedTransport) Flush(ctx context.Context) error {
	size := p.buf.Len()
	buf := p.buffer[:4]
	binary.BigEndian.PutUint32(buf, uint32(size))
	_, err := p.transport.Write(buf)
	if err != nil {
		p.buf.Truncate(0)
		return NewTTransportExceptionFromError(err)
	}
	if size > 0 {
		if n, err := p.buf.WriteTo(p.transport); err != nil {
			print("Error while flushing write buffer of size ", size, " to transport, only wrote ", n, " bytes: ", err.Error(), "\n")
			p.buf.Truncate(0)
			return NewTTransportExceptionFromError(err)
		}
	}
	err = p.transport.Flush(ctx)
	return NewTTransportExceptionFromError(err)
}

func (p *TFramedTransport) readFrameHeader() (uint32, error) {
	buf := p.buffer[:4]
	if _, err := io.ReadFull(p.reader, buf); err != nil {
		return 0, err
	}
	size := binary.BigEndian.Uint32(buf)
	if size < 0 || size > p.maxLength {
		return 0, NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
	}
	return size, nil
}

func (p *TFramedTransport) RemainingBytes() (num_bytes uint64) {
	return uint64(p.frameSize)
}
<commit_msg>THRIFT-5214: Partial rewrite of TFramedTransport<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"io"
)

const DEFAULT_MAX_LENGTH = 16384000

type TFramedTransport struct {
	transport TTransport
	maxLength uint32

	writeBuf bytes.Buffer

	reader  *bufio.Reader
	readBuf bytes.Buffer

	buffer [4]byte
}

type tFramedTransportFactory struct {
	factory   TTransportFactory
	maxLength uint32
}

func NewTFramedTransportFactory(factory TTransportFactory) TTransportFactory {
	return &tFramedTransportFactory{factory: factory, maxLength: DEFAULT_MAX_LENGTH}
}

func NewTFramedTransportFactoryMaxLength(factory TTransportFactory, maxLength uint32) TTransportFactory {
	return &tFramedTransportFactory{factory: factory, maxLength: maxLength}
}

func (p *tFramedTransportFactory) GetTransport(base TTransport) (TTransport, error) {
	tt, err := p.factory.GetTransport(base)
	if err != nil {
		return nil, err
	}
	return NewTFramedTransportMaxLength(tt, p.maxLength), nil
}

func NewTFramedTransport(transport TTransport) *TFramedTransport {
	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: DEFAULT_MAX_LENGTH}
}

func NewTFramedTransportMaxLength(transport TTransport, maxLength uint32) *TFramedTransport {
	return &TFramedTransport{transport: transport, reader: bufio.NewReader(transport), maxLength: maxLength}
}

func (p *TFramedTransport) Open() error {
	return p.transport.Open()
}

func (p *TFramedTransport) IsOpen() bool {
	return p.transport.IsOpen()
}

func (p *TFramedTransport) Close() error {
	return p.transport.Close()
}

func (p *TFramedTransport) Read(buf []byte) (read int, err error) {
	read, err = p.readBuf.Read(buf)
	if err != io.EOF {
		return
	}

	// For bytes.Buffer.Read, EOF would only happen when read is zero,
	// but still, do a sanity check,
	// in case that behavior is changed in a future version of go stdlib.
	// When that happens, just return nil error,
	// and let the caller call Read again to read the next frame.
	if read > 0 {
		return read, nil
	}

	// Reaching here means that the last Read finished the last frame,
	// so we need to read the next frame into readBuf now.
	if err = p.readFrame(); err != nil {
		return read, err
	}
	newRead, err := p.Read(buf[read:])
	return read + newRead, err
}

func (p *TFramedTransport) ReadByte() (c byte, err error) {
	buf := p.buffer[:1]
	_, err = p.Read(buf)
	if err != nil {
		return
	}
	c = buf[0]
	return
}

func (p *TFramedTransport) Write(buf []byte) (int, error) {
	n, err := p.writeBuf.Write(buf)
	return n, NewTTransportExceptionFromError(err)
}

func (p *TFramedTransport) WriteByte(c byte) error {
	return p.writeBuf.WriteByte(c)
}

func (p *TFramedTransport) WriteString(s string) (n int, err error) {
	return p.writeBuf.WriteString(s)
}

func (p *TFramedTransport) Flush(ctx context.Context) error {
	size := p.writeBuf.Len()
	buf := p.buffer[:4]
	binary.BigEndian.PutUint32(buf, uint32(size))
	_, err := p.transport.Write(buf)
	if err != nil {
		p.writeBuf.Reset()
		return NewTTransportExceptionFromError(err)
	}
	if size > 0 {
		if _, err := io.Copy(p.transport, &p.writeBuf); err != nil {
			p.writeBuf.Reset()
			return NewTTransportExceptionFromError(err)
		}
	}
	err = p.transport.Flush(ctx)
	return NewTTransportExceptionFromError(err)
}

func (p *TFramedTransport) readFrame() error {
	buf := p.buffer[:4]
	if _, err := io.ReadFull(p.reader, buf); err != nil {
		return err
	}
	size := binary.BigEndian.Uint32(buf)
	if size < 0 || size > p.maxLength {
		return NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("Incorrect frame size (%d)", size))
	}
	_, err := io.CopyN(&p.readBuf, p.reader, int64(size))
	return NewTTransportExceptionFromError(err)
}

func (p *TFramedTransport) RemainingBytes() (num_bytes uint64) {
	return uint64(p.readBuf.Len())
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package table

import (
	"bytes"
	"encoding/binary"
	"io"
	"math"
	"sync"

	"github.com/AndreasBriese/bbloom"
	"github.com/dgraph-io/badger/y"
)

//var tableSize int64 = 50 << 20
var (
	restartInterval int = 100 // Might want to change this to be based on total size instead of numKeys.
)

var bufPool = sync.Pool{
	New: func() interface{} {
		b := new(bytes.Buffer)
		b.Grow(64 << 20)
		return b
	},
}

type header struct {
	plen uint16 // Overlap with base key.
	klen uint16 // Length of the diff.
	vlen uint16 // Length of value.
	prev uint32 // Offset for the previous key-value pair. The offset is relative to block base offset.
}

// Encode encodes the header.
func (h header) Encode(b []byte) {
	binary.BigEndian.PutUint16(b[0:2], h.plen)
	binary.BigEndian.PutUint16(b[2:4], h.klen)
	binary.BigEndian.PutUint16(b[4:6], h.vlen)
	binary.BigEndian.PutUint32(b[6:10], h.prev)
}

// Decode decodes the header.
func (h *header) Decode(buf []byte) int {
	h.plen = binary.BigEndian.Uint16(buf[0:2])
	h.klen = binary.BigEndian.Uint16(buf[2:4])
	h.vlen = binary.BigEndian.Uint16(buf[4:6])
	h.prev = binary.BigEndian.Uint32(buf[6:10])
	return h.Size()
}

// Size returns size of the header. Currently it's just a constant.
func (h header) Size() int { return 10 }

type TableBuilder struct {
	counter int // Number of keys written for the current block.

	// Typically tens or hundreds of meg. This is for one single file.
	buf *bytes.Buffer

	baseKey    []byte // Base key for the current block.
	baseOffset uint32 // Offset for the current block.

	restarts []uint32 // Base offsets of every block.

	// Tracks offset for the previous key-value pair. Offset is relative to block base offset.
	prevOffset uint32

	keyBuf   *bytes.Buffer
	keyCount int
}

func NewTableBuilder() *TableBuilder {
	return &TableBuilder{
		keyBuf:     bufPool.Get().(*bytes.Buffer),
		buf:        bufPool.Get().(*bytes.Buffer),
		prevOffset: math.MaxUint32, // Used for the first element!
	}
}

// Close closes the TableBuilder. Do not use buf field anymore.
func (b *TableBuilder) Close() {
	b.buf.Reset()
	b.keyBuf.Reset()
	bufPool.Put(b.buf)
	bufPool.Put(b.keyBuf)
}

func (b *TableBuilder) Empty() bool { return b.buf.Len() == 0 }

// keyDiff returns a suffix of newKey that is different from b.baseKey.
func (b TableBuilder) keyDiff(newKey []byte) []byte {
	var i int
	for i = 0; i < len(newKey) && i < len(b.baseKey); i++ {
		if newKey[i] != b.baseKey[i] {
			break
		}
	}
	return newKey[i:]
}

func (b *TableBuilder) addHelper(key []byte, v y.ValueStruct) {
	// Add key to bloom filter.
	var klen [2]byte
	binary.BigEndian.PutUint16(klen[:], uint16(len(key)))
	b.keyBuf.Write(klen[:])
	b.keyBuf.Write(key)
	b.keyCount++

	// diffKey stores the difference of key with baseKey.
	var diffKey []byte
	if len(b.baseKey) == 0 {
		// Make a copy. Builder should not keep references. Otherwise, caller has to be very careful
		// and will have to make copies of keys every time they add to builder, which is even worse.
		b.baseKey = append(b.baseKey[:0], key...)
		diffKey = key
	} else {
		diffKey = b.keyDiff(key)
	}

	h := header{
		plen: uint16(len(key) - len(diffKey)),
		klen: uint16(len(diffKey)),
		vlen: uint16(len(v.Value) + y.MetaSize + y.UserMetaSize + y.CasSize),
		prev: b.prevOffset, // prevOffset is the location of the last key-value added.
	}
	b.prevOffset = uint32(b.buf.Len()) - b.baseOffset // Remember current offset for the next Add call.

	// Layout: header, diffKey, value.
	var hbuf [10]byte
	h.Encode(hbuf[:])
	b.buf.Write(hbuf[:])
	b.buf.Write(diffKey)    // We only need to store the key difference.
	b.buf.WriteByte(v.Meta) // Meta byte precedes actual value.
	b.buf.WriteByte(v.UserMeta)
	var casBytes [y.CasSize]byte
	binary.BigEndian.PutUint64(casBytes[:], v.CASCounter)
	b.buf.Write(casBytes[:])
	b.buf.Write(v.Value)
	b.counter++ // Increment number of keys added for this current block.
}

func (b *TableBuilder) finishBlock() {
	// When we are at the end of the block and Valid=false, and the user wants to do a Prev,
	// we need a dummy header to tell us the offset of the previous key-value pair.
	b.addHelper([]byte{}, y.ValueStruct{})
}

// Add adds a key-value pair to the block.
// If doNotRestart is true, we will not restart even if b.counter >= restartInterval.
func (b *TableBuilder) Add(key []byte, value y.ValueStruct) error {
	if b.counter >= restartInterval {
		b.finishBlock()
		// Start a new block. Initialize the block.
		b.restarts = append(b.restarts, uint32(b.buf.Len()))
		b.counter = 0
		b.baseKey = []byte{}
		b.baseOffset = uint32(b.buf.Len())
		b.prevOffset = math.MaxUint32 // First key-value pair of block has header.prev=MaxInt.
	}
	b.addHelper(key, value)
	return nil // Currently, there is no meaningful error.
}

// FinalSize returns the *rough* final size of the array, counting the header which is not yet written.
// TODO: Look into why there is a discrepancy. I suspect it is because of Write(empty, empty)
// at the end. The diff can vary.
func (b *TableBuilder) ReachedCapacity(cap int64) bool {
	estimateSz := b.buf.Len() + 8 /* empty header */ + 4*len(b.restarts) + 8 // 8 = end of buf offset + len(restarts).
	return int64(estimateSz) > cap
}

// blockIndex generates the block index for the table.
// It is mainly a list of all the block base offsets.
func (b *TableBuilder) blockIndex() []byte {
	// Store the end offset, so we know the length of the final block.
	b.restarts = append(b.restarts, uint32(b.buf.Len()))

	// Add 4 because we want to write out number of restarts at the end.
	sz := 4*len(b.restarts) + 4
	out := make([]byte, sz)
	buf := out
	for _, r := range b.restarts {
		binary.BigEndian.PutUint32(buf[:4], r)
		buf = buf[4:]
	}
	binary.BigEndian.PutUint32(buf[:4], uint32(len(b.restarts)))
	return out
}

// Finish finishes the table by appending the index.
func (b *TableBuilder) Finish() []byte {
	bf := bbloom.New(float64(b.keyCount), 0.01)
	var klen [2]byte
	key := make([]byte, 1024)
	for {
		if _, err := b.keyBuf.Read(klen[:]); err == io.EOF {
			break
		} else if err != nil {
			y.Check(err)
		}
		kl := int(binary.BigEndian.Uint16(klen[:]))
		if cap(key) < kl {
			key = make([]byte, 2*kl)
		}
		key = key[:kl]
		y.Check2(b.keyBuf.Read(key))
		bf.Add(key)
	}

	b.finishBlock() // This will never start a new block.
	index := b.blockIndex()
	b.buf.Write(index)

	// Write bloom filter.
	bdata := bf.JSONMarshal()
	n, err := b.buf.Write(bdata)
	y.Check(err)
	var buf [4]byte
	binary.BigEndian.PutUint32(buf[:], uint32(n))
	b.buf.Write(buf[:])

	return b.buf.Bytes()
}
<commit_msg>Don't use a sync.Pool for table builder buffers. They consume too much memory and cause OOM.<commit_after>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package table

import (
	"bytes"
	"encoding/binary"
	"io"
	"math"

	"github.com/AndreasBriese/bbloom"
	"github.com/dgraph-io/badger/y"
)

//var tableSize int64 = 50 << 20
var (
	restartInterval int = 100 // Might want to change this to be based on total size instead of numKeys.
)

func newBuffer(sz int) *bytes.Buffer {
	b := new(bytes.Buffer)
	b.Grow(sz)
	return b
}

type header struct {
	plen uint16 // Overlap with base key.
	klen uint16 // Length of the diff.
	vlen uint16 // Length of value.
	prev uint32 // Offset for the previous key-value pair. The offset is relative to block base offset.
}

// Encode encodes the header.
func (h header) Encode(b []byte) {
	binary.BigEndian.PutUint16(b[0:2], h.plen)
	binary.BigEndian.PutUint16(b[2:4], h.klen)
	binary.BigEndian.PutUint16(b[4:6], h.vlen)
	binary.BigEndian.PutUint32(b[6:10], h.prev)
}

// Decode decodes the header.
func (h *header) Decode(buf []byte) int {
	h.plen = binary.BigEndian.Uint16(buf[0:2])
	h.klen = binary.BigEndian.Uint16(buf[2:4])
	h.vlen = binary.BigEndian.Uint16(buf[4:6])
	h.prev = binary.BigEndian.Uint32(buf[6:10])
	return h.Size()
}

// Size returns size of the header. Currently it's just a constant.
func (h header) Size() int { return 10 }

type TableBuilder struct {
	counter int // Number of keys written for the current block.

	// Typically tens or hundreds of meg. This is for one single file.
	buf *bytes.Buffer

	baseKey    []byte // Base key for the current block.
	baseOffset uint32 // Offset for the current block.

	restarts []uint32 // Base offsets of every block.

	// Tracks offset for the previous key-value pair. Offset is relative to block base offset.
	prevOffset uint32

	keyBuf   *bytes.Buffer
	keyCount int
}

func NewTableBuilder() *TableBuilder {
	return &TableBuilder{
		keyBuf:     newBuffer(32 << 20),
		buf:        newBuffer(64 << 20),
		prevOffset: math.MaxUint32, // Used for the first element!
	}
}

// Close closes the TableBuilder.
func (b *TableBuilder) Close() {}

func (b *TableBuilder) Empty() bool { return b.buf.Len() == 0 }

// keyDiff returns a suffix of newKey that is different from b.baseKey.
func (b TableBuilder) keyDiff(newKey []byte) []byte {
	var i int
	for i = 0; i < len(newKey) && i < len(b.baseKey); i++ {
		if newKey[i] != b.baseKey[i] {
			break
		}
	}
	return newKey[i:]
}

func (b *TableBuilder) addHelper(key []byte, v y.ValueStruct) {
	// Add key to bloom filter.
	var klen [2]byte
	binary.BigEndian.PutUint16(klen[:], uint16(len(key)))
	b.keyBuf.Write(klen[:])
	b.keyBuf.Write(key)
	b.keyCount++

	// diffKey stores the difference of key with baseKey.
	var diffKey []byte
	if len(b.baseKey) == 0 {
		// Make a copy. Builder should not keep references. Otherwise, caller has to be very careful
		// and will have to make copies of keys every time they add to builder, which is even worse.
		b.baseKey = append(b.baseKey[:0], key...)
		diffKey = key
	} else {
		diffKey = b.keyDiff(key)
	}

	h := header{
		plen: uint16(len(key) - len(diffKey)),
		klen: uint16(len(diffKey)),
		vlen: uint16(len(v.Value) + y.MetaSize + y.UserMetaSize + y.CasSize),
		prev: b.prevOffset, // prevOffset is the location of the last key-value added.
	}
	b.prevOffset = uint32(b.buf.Len()) - b.baseOffset // Remember current offset for the next Add call.

	// Layout: header, diffKey, value.
	var hbuf [10]byte
	h.Encode(hbuf[:])
	b.buf.Write(hbuf[:])
	b.buf.Write(diffKey)    // We only need to store the key difference.
	b.buf.WriteByte(v.Meta) // Meta byte precedes actual value.
	b.buf.WriteByte(v.UserMeta)
	var casBytes [y.CasSize]byte
	binary.BigEndian.PutUint64(casBytes[:], v.CASCounter)
	b.buf.Write(casBytes[:])
	b.buf.Write(v.Value)
	b.counter++ // Increment number of keys added for this current block.
}

func (b *TableBuilder) finishBlock() {
	// When we are at the end of the block and Valid=false, and the user wants to do a Prev,
	// we need a dummy header to tell us the offset of the previous key-value pair.
	b.addHelper([]byte{}, y.ValueStruct{})
}

// Add adds a key-value pair to the block.
// If doNotRestart is true, we will not restart even if b.counter >= restartInterval.
func (b *TableBuilder) Add(key []byte, value y.ValueStruct) error {
	if b.counter >= restartInterval {
		b.finishBlock()
		// Start a new block. Initialize the block.
		b.restarts = append(b.restarts, uint32(b.buf.Len()))
		b.counter = 0
		b.baseKey = []byte{}
		b.baseOffset = uint32(b.buf.Len())
		b.prevOffset = math.MaxUint32 // First key-value pair of block has header.prev=MaxInt.
	}
	b.addHelper(key, value)
	return nil // Currently, there is no meaningful error.
}

// FinalSize returns the *rough* final size of the array, counting the header which is not yet written.
// TODO: Look into why there is a discrepancy. I suspect it is because of Write(empty, empty)
// at the end. The diff can vary.
func (b *TableBuilder) ReachedCapacity(cap int64) bool {
	estimateSz := b.buf.Len() + 8 /* empty header */ + 4*len(b.restarts) + 8 // 8 = end of buf offset + len(restarts).
	return int64(estimateSz) > cap
}

// blockIndex generates the block index for the table.
// It is mainly a list of all the block base offsets.
func (b *TableBuilder) blockIndex() []byte {
	// Store the end offset, so we know the length of the final block.
	b.restarts = append(b.restarts, uint32(b.buf.Len()))

	// Add 4 because we want to write out number of restarts at the end.
	sz := 4*len(b.restarts) + 4
	out := make([]byte, sz)
	buf := out
	for _, r := range b.restarts {
		binary.BigEndian.PutUint32(buf[:4], r)
		buf = buf[4:]
	}
	binary.BigEndian.PutUint32(buf[:4], uint32(len(b.restarts)))
	return out
}

// Finish finishes the table by appending the index.
func (b *TableBuilder) Finish() []byte {
	bf := bbloom.New(float64(b.keyCount), 0.01)
	var klen [2]byte
	key := make([]byte, 1024)
	for {
		if _, err := b.keyBuf.Read(klen[:]); err == io.EOF {
			break
		} else if err != nil {
			y.Check(err)
		}
		kl := int(binary.BigEndian.Uint16(klen[:]))
		if cap(key) < kl {
			key = make([]byte, 2*kl)
		}
		key = key[:kl]
		y.Check2(b.keyBuf.Read(key))
		bf.Add(key)
	}

	b.finishBlock() // This will never start a new block.
	index := b.blockIndex()
	b.buf.Write(index)

	// Write bloom filter.
	bdata := bf.JSONMarshal()
	n, err := b.buf.Write(bdata)
	y.Check(err)
	var buf [4]byte
	binary.BigEndian.PutUint32(buf[:], uint32(n))
	b.buf.Write(buf[:])

	return b.buf.Bytes()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode_test

import (
	"fmt"
	"math"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
)

func TestFile(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const uid = 123
const gid = 456

const fileInodeID = 17
const fileInodeName = "foo/bar"
const fileMode os.FileMode = 0641

type FileTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.SimulatedClock

	initialContents string
	backingObj      *gcs.Object

	in *inode.FileInode
}

var _ SetUpInterface = &FileTest{}
var _ TearDownInterface = &FileTest{}

func init() { RegisterTestSuite(&FileTest{}) }

func (t *FileTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.leaser = lease.NewFileLeaser("", math.MaxInt64)
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")

	// Set up the backing object.
	var err error

	t.initialContents = "taco"
	t.backingObj, err = gcsutil.CreateObject(
		t.ctx,
		t.bucket,
		fileInodeName,
		t.initialContents)

	AssertEq(nil, err)

	// Create the inode.
	t.in = inode.NewFileInode(
		fileInodeID,
		t.backingObj,
		uid,
		gid,
		fileMode,
		math.MaxUint64, // GCS chunk size
		false,          // Support nlink
		t.bucket,
		t.leaser,
		&t.clock)

	t.in.Lock()
}

func (t *FileTest) TearDown() {
	t.in.Unlock()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileTest) ID() {
	ExpectEq(fileInodeID, t.in.ID())
}

func (t *FileTest) Name() {
	ExpectEq(fileInodeName, t.in.Name())
}

func (t *FileTest) InitialSourceGeneration() {
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())
}

func (t *FileTest) InitialAttributes() {
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len(t.initialContents), attrs.Size)
	ExpectEq(1, attrs.Nlink)
	ExpectEq(uid, attrs.Uid)
	ExpectEq(gid, attrs.Gid)
	ExpectEq(fileMode, attrs.Mode)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated))
}

func (t *FileTest) Read() {
	AssertEq("taco", t.initialContents)

	// Make several reads, checking the expected contents. We should never get an
	// EOF error, since fuseops.ReadFileOp is not supposed to see those.
	testCases := []struct {
		offset   int64
		size     int
		expected string
	}{
		{0, 1, "t"},
		{0, 2, "ta"},
		{0, 3, "tac"},
		{0, 4, "taco"},
		{0, 5, "taco"},

		{1, 1, "a"},
		{1, 2, "ac"},
		{1, 3, "aco"},
		{1, 4, "aco"},

		{3, 1, "o"},
		{3, 2, "o"},

		// Empty ranges
		{0, 0, ""},
		{3, 0, ""},
		{4, 0, ""},
		{4, 1, ""},
		{5, 0, ""},
		{5, 1, ""},
		{5, 2, ""},
	}

	for _, tc := range testCases {
		desc := fmt.Sprintf("offset: %d, size: %d", tc.offset, tc.size)

		data, err := t.in.Read(t.ctx, tc.offset, tc.size)
		AssertEq(nil, err, "%s", desc)
		ExpectEq(tc.expected, string(data), "%s", desc)
	}
}

func (t *FileTest) Write() {
	var data []byte
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	// Add some data at the end.
	t.clock.AdvanceTime(time.Second)
	writeTime := t.clock.Now()

	err = t.in.Write(t.ctx, []byte("burrito"), 4)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read back the content.
	data, err = t.in.Read(t.ctx, 0, 1024)
	AssertEq(nil, err)
	ExpectEq("pacoburrito", string(data))

	// Check attributes.
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("pacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime))
}

func (t *FileTest) Truncate() {
	var attrs fuseops.InodeAttributes
	var data []byte
	var err error

	AssertEq("taco", t.initialContents)

	// Truncate downward.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()

	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read the contents.
	data, err = t.in.Read(t.ctx, 0, 1024)
	AssertEq(nil, err)
	ExpectEq("ta", string(data))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("ta"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime))
}

func (t *FileTest) Sync_NotClobbered() {
	var attrs fuseops.InodeAttributes
	var err error

	// Truncate downward.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()

	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(2, o.Size)

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(2, attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime))
}

func (t *FileTest) Sync_Clobbered() {
	var err error

	// Truncate downward.
	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Clobber the backing object.
	newObj, err := gcsutil.CreateObject(t.ctx, t.bucket, t.in.Name(), "burrito")
	AssertEq(nil, err)

	// Sync. The call should succeed, but nothing should change.
	err = t.in.Sync(t.ctx)

	AssertEq(nil, err)
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())

	// The object in the bucket should not have been changed.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(newObj.Generation, o.Generation)
	ExpectEq(newObj.Size, o.Size)
}
<commit_msg>Fixed a build error.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode_test

import (
	"fmt"
	"math"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
)

func TestFile(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const uid = 123
const gid = 456

const fileInodeID = 17
const fileInodeName = "foo/bar"
const fileMode os.FileMode = 0641

type FileTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.SimulatedClock

	initialContents string
	backingObj      *gcs.Object

	in *inode.FileInode
}

var _ SetUpInterface = &FileTest{}
var _ TearDownInterface = &FileTest{}

func init() { RegisterTestSuite(&FileTest{}) }

func (t *FileTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.leaser = lease.NewFileLeaser("", math.MaxInt32, math.MaxInt64)
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")

	// Set up the backing object.
	var err error

	t.initialContents = "taco"
	t.backingObj, err = gcsutil.CreateObject(
		t.ctx,
		t.bucket,
		fileInodeName,
		t.initialContents)

	AssertEq(nil, err)

	// Create the inode.
	t.in = inode.NewFileInode(
		fileInodeID,
		t.backingObj,
		uid,
		gid,
		fileMode,
		math.MaxUint64, // GCS chunk size
		false,          // Support nlink
		t.bucket,
		t.leaser,
		&t.clock)

	t.in.Lock()
}

func (t *FileTest) TearDown() {
	t.in.Unlock()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileTest) ID() {
	ExpectEq(fileInodeID, t.in.ID())
}

func (t *FileTest) Name() {
	ExpectEq(fileInodeName, t.in.Name())
}

func (t *FileTest) InitialSourceGeneration() {
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())
}

func (t *FileTest) InitialAttributes() {
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len(t.initialContents), attrs.Size)
	ExpectEq(1, attrs.Nlink)
	ExpectEq(uid, attrs.Uid)
	ExpectEq(gid, attrs.Gid)
	ExpectEq(fileMode, attrs.Mode)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated))
}

func (t *FileTest) Read() {
	AssertEq("taco", t.initialContents)

	// Make several reads, checking the expected contents. We should never get an
	// EOF error, since fuseops.ReadFileOp is not supposed to see those.
	testCases := []struct {
		offset   int64
		size     int
		expected string
	}{
		{0, 1, "t"},
		{0, 2, "ta"},
		{0, 3, "tac"},
		{0, 4, "taco"},
		{0, 5, "taco"},

		{1, 1, "a"},
		{1, 2, "ac"},
		{1, 3, "aco"},
		{1, 4, "aco"},

		{3, 1, "o"},
		{3, 2, "o"},

		// Empty ranges
		{0, 0, ""},
		{3, 0, ""},
		{4, 0, ""},
		{4, 1, ""},
		{5, 0, ""},
		{5, 1, ""},
		{5, 2, ""},
	}

	for _, tc := range testCases {
		desc := fmt.Sprintf("offset: %d, size: %d", tc.offset, tc.size)

		data, err := t.in.Read(t.ctx, tc.offset, tc.size)
		AssertEq(nil, err, "%s", desc)
		ExpectEq(tc.expected, string(data), "%s", desc)
	}
}

func (t *FileTest) Write() {
	var data []byte
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	// Add some data at the end.
	t.clock.AdvanceTime(time.Second)
	writeTime := t.clock.Now()

	err = t.in.Write(t.ctx, []byte("burrito"), 4)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read back the content.
	data, err = t.in.Read(t.ctx, 0, 1024)
	AssertEq(nil, err)
	ExpectEq("pacoburrito", string(data))

	// Check attributes.
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("pacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime))
}

func (t *FileTest) Truncate() {
	var attrs fuseops.InodeAttributes
	var data []byte
	var err error

	AssertEq("taco", t.initialContents)

	// Truncate downward.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()

	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read the contents.
	data, err = t.in.Read(t.ctx, 0, 1024)
	AssertEq(nil, err)
	ExpectEq("ta", string(data))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("ta"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime))
}

func (t *FileTest) Sync_NotClobbered() {
	var attrs fuseops.InodeAttributes
	var err error

	// Truncate downward.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()

	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(2, o.Size)

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(2, attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime))
}

func (t *FileTest) Sync_Clobbered() {
	var err error

	// Truncate downward.
	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Clobber the backing object.
	newObj, err := gcsutil.CreateObject(t.ctx, t.bucket, t.in.Name(), "burrito")
	AssertEq(nil, err)

	// Sync. The call should succeed, but nothing should change.
	err = t.in.Sync(t.ctx)

	AssertEq(nil, err)
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())

	// The object in the bucket should not have been changed.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(newObj.Generation, o.Generation)
	ExpectEq(newObj.Size, o.Size)
}
<|endoftext|>


<commit_before>package backends

import (
	"io"
	"fmt"
	"github.com/docker/libswarm/beam"
	beamutils "github.com/docker/libswarm/beam/utils"
	"strings"
	"sync"
	"time"
)

// New returns a new engine, with all backends
// registered but not activated.
// To activate a backend, call a job on the resulting
// engine, named after the desired backend.
//
// Example: `New().Job("debug").Run()`
func New() beam.Sender {
	backends := beamutils.NewHub()
	backends.RegisterName("cd", func(msg *beam.Message, in beam.Receiver, out beam.Sender, next beam.Sender) (bool, error) {
		return false, fmt.Errorf("no such backend: %s\n", strings.Join(msg.Args, " "))
	})
	backends.RegisterName("cd", func(msg *beam.Message, in beam.Receiver, out beam.Sender, next beam.Sender) (bool, error) {
		if len(msg.Args) > 0 && msg.Args[0] == "debug" {
			debugr, debugw, err := out.Send(&beam.Message{Name: "register"}, beam.R|beam.W)
			if err != nil {
				return false, err
			}
			go func() {
				for {
					msg, msgr, msgw, err := debugr.Receive(beam.R | beam.W)
					if err == io.EOF {
						return
					}
					if err != nil {
						return
					}
					fmt.Printf("[DEBUG] %s %s\n", msg.Name, strings.Join(msg.Args, " "))
					// FIXME: goroutine?
					Splice(debugw, msg, msgr, msgw)
				}
			}()
			return false, nil
		}
		return true, nil
	})
	backends.RegisterName("fakeclient", func(msg *beam.Message, in beam.Receiver, out beam.Sender, next beam.Sender) (bool, error) {
		backends.RegisterTask(func(r beam.Receiver, w beam.Sender) error {
			for {
				time.Sleep(1 * time.Second)
				_, _, err := w.Send(&beam.Message{Name: "log", Args: []string{"fake client reporting for duty"}}, 0)
				if err != nil {
					return err
				}
				containersR, _, err := w.Send(&beam.Message{Name: "containers"}, beam.R)
				if err != nil {
					return err
				}
				go beamutils.Copy(beam.NopSender{}, containersR)
			}
		})
		return true, nil
	})
	return backends
}

func Splice(dst beam.Sender, msg *beam.Message, r beam.Receiver, w beam.Sender) error {
	dstR, dstW, err := dst.Send(msg, beam.R|beam.W)
	if err != nil {
		return err
	}
	defer dstW.Close()
	var tasks sync.WaitGroup
	_copy := func(dst beam.Sender, src beam.Receiver) {
		tasks.Add(1)
		go func() {
			defer tasks.Done()
			beamutils.Copy(dst, src)
		}()
	}
	_copy(dstW, r)
	_copy(w, dstR)
	tasks.Wait()
	return nil
}
<commit_msg>Splice: close destination when copy is over<commit_after>package backends

import (
	"io"
	"fmt"
	"github.com/docker/libswarm/beam"
	beamutils "github.com/docker/libswarm/beam/utils"
	"strings"
	"sync"
	"time"
)

// New returns a new engine, with all backends
// registered but not activated.
// To activate a backend, call a job on the resulting
// engine, named after the desired backend.
//
// Example: `New().Job("debug").Run()`
func New() beam.Sender {
	backends := beamutils.NewHub()
	backends.RegisterName("cd", func(msg *beam.Message, in beam.Receiver, out beam.Sender, next beam.Sender) (bool, error) {
		return false, fmt.Errorf("no such backend: %s\n", strings.Join(msg.Args, " "))
	})
	backends.RegisterName("cd", func(msg *beam.Message, in beam.Receiver, out beam.Sender, next beam.Sender) (bool, error) {
		if len(msg.Args) > 0 && msg.Args[0] == "debug" {
			debugr, debugw, err := out.Send(&beam.Message{Name: "register"}, beam.R|beam.W)
			if err != nil {
				return false, err
			}
			go func() {
				for {
					msg, msgr, msgw, err := debugr.Receive(beam.R | beam.W)
					if err == io.EOF {
						return
					}
					if err != nil {
						return
					}
					fmt.Printf("[DEBUG] %s %s\n", msg.Name, strings.Join(msg.Args, " "))
					// FIXME: goroutine?
					Splice(debugw, msg, msgr, msgw)
				}
			}()
			return false, nil
		}
		return true, nil
	})
	backends.RegisterName("fakeclient", func(msg *beam.Message, in beam.Receiver, out beam.Sender, next beam.Sender) (bool, error) {
		backends.RegisterTask(func(r beam.Receiver, w beam.Sender) error {
			for {
				time.Sleep(1 * time.Second)
				_, _, err := w.Send(&beam.Message{Name: "log", Args: []string{"fake client reporting for duty"}}, 0)
				if err != nil {
					return err
				}
				containersR, _, err := w.Send(&beam.Message{Name: "containers"}, beam.R)
				if err != nil {
					return err
				}
				go beamutils.Copy(beam.NopSender{}, containersR)
			}
		})
		return true, nil
	})
	return backends
}

func Splice(dst beam.Sender, msg *beam.Message, r beam.Receiver, w beam.Sender) error {
	dstR, dstW, err := dst.Send(msg, beam.R|beam.W)
	if err != nil {
		return err
	}
	defer dstW.Close()
	var tasks sync.WaitGroup
	_copy := func(dst beam.Sender, src beam.Receiver) {
		tasks.Add(1)
		go func() {
			defer tasks.Done()
			beamutils.Copy(dst, src)
			dst.Close()
		}()
	}
	_copy(dstW, r)
	_copy(w, dstR)
	tasks.Wait()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 单进程高速缓存.
package gcache

import (
    "sync"
    "time"
    "math"
    "gitee.com/johng/gf/g/os/gtime"
    "gitee.com/johng/gf/g/container/gset"
    "gitee.com/johng/gf/g/container/gqueue"
    "gitee.com/johng/gf/g/container/gtype"
)

const (
    gDEFAULT_MAX_EXPIRE = 9223372036854  // 当数据不过期时，默认设置的过期属性值，相当于：math.MaxInt64/1000000
)

// 缓存对象
type Cache struct {
    dmu        sync.RWMutex              // data锁
    emu        sync.RWMutex              // ekmap锁
    smu        sync.RWMutex              // eksets锁
    lru        *_Lru                     // LRU缓存限制
    cap        *gtype.Int                // 控制缓存池大小，超过大小则按照LRU算法进行缓存过期处理(默认为0表示不进行限制)
    data       map[string]_CacheItem     // 缓存数据(所有的缓存数据存放哈希表)
    ekmap      map[string]int64          // 键名对应的分组过期时间(用于相同键名过期时间快速更新)
    eksets     map[int64]*gset.StringSet // 分组过期时间对应的键名列表(用于自动过期快速删除)
    eventQueue *gqueue.Queue             // 异步处理队列
    stopEvents chan struct{}             // 关闭时间通知
}

// 缓存数据项
type _CacheItem struct {
    v interface{} // 缓存键值
    e int64       // 过期时间
}

// 异步队列数据项
type _EventItem struct {
    k string      // 键名
    e int64       // 过期时间
}

// 全局缓存管理对象
var cache *Cache = New()

// Cache对象按照缓存键名首字母做了分组
func New() *Cache {
    c := &Cache {
        lru        : newLru(),
        cap        : gtype.NewInt(),
        data       : make(map[string]_CacheItem),
        ekmap      : make(map[string]int64),
        eksets     : make(map[int64]*gset.StringSet),
        eventQueue : gqueue.New(),
        stopEvents : make(chan struct{}, 2),
    }
    go c.autoSyncLoop()
    go c.autoClearLoop()
    return c
}

// 设置缓存池大小，内部依靠LRU算法进行缓存淘汰处理
func SetCap(cap int) {
    cache.cap.Set(cap)
}

// (使用全局KV缓存对象)设置kv缓存键值对，过期时间单位为毫秒
func Set(key string, value interface{}, expire int64)  {
    cache.Set(key, value, expire)
}

// (使用全局KV缓存对象)批量设置kv缓存键值对，过期时间单位为毫秒
func BatchSet(data map[string]interface{}, expire int64)  {
    cache.BatchSet(data, expire)
}

// (使用全局KV缓存对象)获取指定键名的值
func Get(key string) interface{} {
    return cache.Get(key)
}

// (使用全局KV缓存对象)删除指定键值对
func Remove(key string) {
    cache.Remove(key)
}

// (使用全局KV缓存对象)批量删除指定键值对
func BatchRemove(keys []string) {
    cache.BatchRemove(keys)
}

// 获得所有的键名，组成字符串数组返回
func Keys() []string {
    return cache.Keys()
}

// 获得所有的值，组成数组返回
func Values() []interface{} {
    return cache.Values()
}

// 获得缓存对象的键值对数量
func Size() int {
    return cache.Size()
}

// 设置缓存池大小，内部依靠LRU算法进行缓存淘汰处理
func (c *Cache) SetCap(cap int) {
    c.cap.Set(cap)
}

// 计算过期缓存的键名(将毫秒换算成秒的整数毫秒)
func (c *Cache) makeExpireKey(expire int64) int64 {
    return int64(math.Ceil(float64(expire/1000) + 1)*1000)
}

// 获取一个过期键名存放Set,如果没有则返回nil
func (c *Cache) getExpireSet(expire int64) *gset.StringSet {
    c.smu.RLock()
    if ekset, ok := c.eksets[expire]; ok {
        c.smu.RUnlock()
        return ekset
    }
    c.smu.RUnlock()
    return nil
}

// 获取或者创建一个过期键名存放Set(由于是异步单线程执行，因此不会出现创建set时的覆盖问题)
func (c *Cache) getOrNewExpireSet(expire int64) *gset.StringSet {
    if ekset := c.getExpireSet(expire); ekset == nil {
        set := gset.NewStringSet()
        c.smu.Lock()
        c.eksets[expire] = set
        c.smu.Unlock()
        return set
    } else {
        return ekset
    }
}

// 设置kv缓存键值对，过期时间单位为毫秒，expire<=0表示不过期
func (c *Cache) Set(key string, value interface{}, expire int64) {
    var e int64
    if expire != 0 {
        e = gtime.Millisecond() + int64(expire)
    } else {
        e = gDEFAULT_MAX_EXPIRE
    }
    c.dmu.Lock()
    c.data[key] = _CacheItem{v : value, e : e}
    c.dmu.Unlock()
    c.eventQueue.PushBack(_EventItem{k : key, e : e})
}

// 批量设置
func (c *Cache) BatchSet(data map[string]interface{}, expire int64)  {
    var e int64
    if expire != 0 {
        e = gtime.Millisecond() + int64(expire)
    } else {
        e = gDEFAULT_MAX_EXPIRE
    }
    for k, v := range data {
        c.dmu.Lock()
        c.data[k] = _CacheItem{v: v, e: e}
        c.dmu.Unlock()
        c.eventQueue.PushBack(_EventItem{k: k, e:e})
    }
}

// 获取指定键名的值
func (c *Cache) Get(key string) interface{} {
    c.dmu.RLock()
    item, ok := c.data[key]
    c.dmu.RUnlock()
    if ok {
        if item.e > gtime.Millisecond() {
            return item.v
        }
    }
    return nil
}

// 删除指定键值对
func (c *Cache) Remove(key string) {
    c.Set(key, nil, -1000)
}

// 批量删除键值对
func (c *Cache) BatchRemove(keys []string) {
    for _, key := range keys {
        c.dmu.Lock()
        c.data[key] = _CacheItem{v: nil, e: -1000}
        c.dmu.Unlock()
        c.eventQueue.PushBack(_EventItem{k: key, e: -1000})
    }
}

// 获得所有的键名，组成字符串数组返回
func (c *Cache) Keys() []string {
    keys := make([]string, 0)
    c.dmu.RLock()
    for k, _ := range c.data {
        keys = append(keys, k)
    }
    c.dmu.RUnlock()
    return keys
}

// 获得所有的值，组成数组返回
func (c *Cache) Values() []interface{} {
    values := make([]interface{}, 0)
    c.dmu.RLock()
    for _, v := range c.data {
        values = append(values, v)
    }
    c.dmu.RUnlock()
    return values
}

// 获得缓存对象的键值对数量
func (c *Cache) Size() int {
    c.dmu.RLock()
    length := len(c.data)
    c.dmu.RUnlock()
    return length
}

// 删除缓存对象
func (c *Cache) Close()  {
    c.stopEvents <- struct{}{}
    c.eventQueue.Close()
    c.lru.Close()
}

// 数据自动同步循环
func (c *Cache) autoSyncLoop() {
    for {
        if r := c.eventQueue.PopFront(); r != nil {
            item := r.(_EventItem)
            newe := c.makeExpireKey(item.e)
            // 查询键名是否已经存在过期时间
            c.emu.RLock()
            olde, ok := c.ekmap[item.k];
            c.emu.RUnlock()
            // 是否需要删除旧的过期时间map中对应的键名
            if ok && newe != olde {
                if ekset := c.getExpireSet(olde); ekset != nil {
                    ekset.Remove(item.k)
                }
            }
            c.getOrNewExpireSet(newe).Add(item.k)
            // 重新设置对应键名的过期时间
            c.emu.Lock()
            c.ekmap[item.k] = newe
            c.emu.Unlock()
            // LRU操作记录(只有新增和修改操作才会记录到LRU管理对象中，删除不会)
            if newe >= olde {
                c.lru.Push(item.k)
            }
        } else {
            return
        }
    }

}

// LRU缓存淘汰处理+自动清理过期键值对
// 每隔1秒清除过去3秒的键值对数据
func (c *Cache) autoClearLoop() {
    for {
        select {
            case <- c.stopEvents:
                return
            default:
                // 缓存过期处理
                ek  := c.makeExpireKey(gtime.Millisecond())
                eks := []int64{ek - 2000, ek - 3000, ek - 4000}
                for _, v := range eks {
                    if ekset := c.getExpireSet(v); ekset != nil {
                        ekset.Iterator(c.clearByKey)
                    }
                    // 删除异步处理键名set
                    c.smu.Lock()
                    delete(c.eksets, v)
                    c.smu.Unlock()
                }
                // LRU缓存淘汰处理
                if c.cap.Val() > 0 {
                    for i := c.Size() - c.cap.Val(); i > 0; i-- {
                        if s := c.lru.Pop(); s != "" {
                            c.clearByKey(s)
                        }
                    }
                }
                time.Sleep(time.Second)
        }
    }
}

// 删除对应键名的缓存数据
func (c *Cache) clearByKey(key string) {
    // 删除缓存数据
    c.dmu.Lock()
    delete(c.data,  key)
    c.dmu.Unlock()

    // 删除异步处理数据项，并更新缓存的内存使用大小记录值
    c.emu.Lock()
    delete(c.ekmap, key)
    c.emu.Unlock()

    // 删除LRU管理对象中指定键名
    c.lru.Remove(key)
}
<commit_msg>gcache增加内存锁<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 单进程高速缓存.
package gcache

import (
    "sync"
    "time"
    "math"
    "gitee.com/johng/gf/g/os/gtime"
    "gitee.com/johng/gf/g/container/gset"
    "gitee.com/johng/gf/g/container/gqueue"
    "gitee.com/johng/gf/g/container/gtype"
)

const (
    gDEFAULT_MAX_EXPIRE = 9223372036854  // 当数据不过期时，默认设置的过期属性值，相当于：math.MaxInt64/1000000
)

// 缓存对象
type Cache struct {
    dmu        sync.RWMutex              // data锁
    emu        sync.RWMutex              // ekmap锁
    smu        sync.RWMutex              // eksets锁
    lru        *_Lru                     // LRU缓存限制
    cap        *gtype.Int                // 控制缓存池大小，超过大小则按照LRU算法进行缓存过期处理(默认为0表示不进行限制)
    data       map[string]_CacheItem     // 缓存数据(所有的缓存数据存放哈希表)
    ekmap      map[string]int64          // 键名对应的分组过期时间(用于相同键名过期时间快速更新)
    eksets     map[int64]*gset.StringSet // 分组过期时间对应的键名列表(用于自动过期快速删除)
    eventQueue *gqueue.Queue             // 异步处理队列
    stopEvents chan struct{}             // 关闭时间通知
}

// 缓存数据项
type _CacheItem struct {
    v interface{} // 缓存键值
    e int64       // 过期时间
}

// 异步队列数据项
type _EventItem struct {
    k string      // 键名
    e int64       // 过期时间
}

// 全局缓存管理对象
var cache *Cache = New()

// Cache对象按照缓存键名首字母做了分组
func New() *Cache {
    c := &Cache {
        lru        : newLru(),
        cap        : gtype.NewInt(),
        data       : make(map[string]_CacheItem),
        ekmap      : make(map[string]int64),
        eksets     : make(map[int64]*gset.StringSet),
        eventQueue : gqueue.New(),
        stopEvents : make(chan struct{}, 2),
    }
    go c.autoSyncLoop()
    go c.autoClearLoop()
    return c
}

// 设置缓存池大小，内部依靠LRU算法进行缓存淘汰处理
func SetCap(cap int) {
    cache.cap.Set(cap)
}

// (使用全局KV缓存对象)设置kv缓存键值对，过期时间单位为毫秒
func Set(key string, value interface{}, expire int64)  {
    cache.Set(key, value, expire)
}

// (使用全局KV缓存对象)批量设置kv缓存键值对，过期时间单位为毫秒
func BatchSet(data map[string]interface{}, expire int64)  {
    cache.BatchSet(data, expire)
}

// (使用全局KV缓存对象)获取指定键名的值
func Get(key string) interface{} {
    return cache.Get(key)
}

// (使用全局KV缓存对象)删除指定键值对
func Remove(key string) {
    cache.Remove(key)
}

// (使用全局KV缓存对象)批量删除指定键值对
func BatchRemove(keys []string) {
    cache.BatchRemove(keys)
}

// 基于内存缓存的锁，锁成功返回true，失败返回false，当失败时表示有其他的锁存在
func Lock(key string, expire int64) bool {
    if v := cache.Get(key); v != nil {
        return false
    }
    cache.Set(key, struct {}{}, expire)
    return true
}

// 解除基于内存缓存的锁
func Unlock(key string) {
    cache.Remove(key)
}

// 获得所有的键名，组成字符串数组返回
func Keys() []string {
    return cache.Keys()
}

// 获得所有的值，组成数组返回
func Values() []interface{} {
    return cache.Values()
}

// 获得缓存对象的键值对数量
func Size() int {
    return cache.Size()
}

// 设置缓存池大小，内部依靠LRU算法进行缓存淘汰处理
func (c *Cache) SetCap(cap int) {
    c.cap.Set(cap)
}

// 计算过期缓存的键名(将毫秒换算成秒的整数毫秒)
func (c *Cache) makeExpireKey(expire int64) int64 {
    return int64(math.Ceil(float64(expire/1000) + 1)*1000)
}

// 获取一个过期键名存放Set,如果没有则返回nil
func (c *Cache) getExpireSet(expire int64) *gset.StringSet {
    c.smu.RLock()
    if ekset, ok := c.eksets[expire]; ok {
        c.smu.RUnlock()
        return ekset
    }
    c.smu.RUnlock()
    return nil
}

// 获取或者创建一个过期键名存放Set(由于是异步单线程执行，因此不会出现创建set时的覆盖问题)
func (c *Cache) getOrNewExpireSet(expire int64) *gset.StringSet {
    if ekset := c.getExpireSet(expire); ekset == nil {
        set := gset.NewStringSet()
        c.smu.Lock()
        c.eksets[expire] = set
        c.smu.Unlock()
        return set
    } else {
        return ekset
    }
}

// 设置kv缓存键值对，过期时间单位为毫秒，expire<=0表示不过期
func (c *Cache) Set(key string, value interface{}, expire int64) {
    var e int64
    if expire != 0 {
        e = gtime.Millisecond() + int64(expire)
    } else {
        e = gDEFAULT_MAX_EXPIRE
    }
    c.dmu.Lock()
    c.data[key] = _CacheItem{v : value, e : e}
    c.dmu.Unlock()
    c.eventQueue.PushBack(_EventItem{k : key, e : e})
}

// 批量设置
func (c *Cache) BatchSet(data map[string]interface{}, expire int64)  {
    var e int64
    if expire != 0 {
        e = gtime.Millisecond() + int64(expire)
    } else {
        e = gDEFAULT_MAX_EXPIRE
    }
    for k, v := range data {
        c.dmu.Lock()
        c.data[k] = _CacheItem{v: v, e: e}
        c.dmu.Unlock()
        c.eventQueue.PushBack(_EventItem{k: k, e:e})
    }
}

// 基于内存缓存的锁，锁成功返回true，失败返回false，当失败时表示有其他的锁存在
func (c *Cache) Lock(key string, expire int64) bool {
    if v := c.Get(key); v != nil {
        return false
    }
    c.Set(key, struct {}{}, expire)
    return true
}

// 解除基于内存缓存的锁
func (c *Cache) Unlock(key string) {
    c.Remove(key)
}

// 获取指定键名的值
func (c *Cache) Get(key string) interface{} {
    c.dmu.RLock()
    item, ok := c.data[key]
    c.dmu.RUnlock()
    if ok {
        if item.e > gtime.Millisecond() {
            return item.v
        }
    }
    return nil
}

// 删除指定键值对
func (c *Cache) Remove(key string) {
    c.Set(key, nil, -1000)
}

// 批量删除键值对
func (c *Cache) BatchRemove(keys []string) {
    for _, key := range keys {
        c.dmu.Lock()
        c.data[key] = _CacheItem{v: nil, e: -1000}
        c.dmu.Unlock()
        c.eventQueue.PushBack(_EventItem{k: key, e: -1000})
    }
}

// 获得所有的键名，组成字符串数组返回
func (c *Cache) Keys() []string {
    keys := make([]string, 0)
    c.dmu.RLock()
    for k, _ := range c.data {
        keys = append(keys, k)
    }
    c.dmu.RUnlock()
    return keys
}

// 获得所有的值，组成数组返回
func (c *Cache) Values() []interface{} {
    values := make([]interface{}, 0)
    c.dmu.RLock()
    for _, v := range c.data {
        values = append(values, v)
    }
    c.dmu.RUnlock()
    return values
}

// 获得缓存对象的键值对数量
func (c *Cache) Size() int {
    c.dmu.RLock()
    length := len(c.data)
    c.dmu.RUnlock()
    return length
}

// 删除缓存对象
func (c *Cache) Close()  {
    c.stopEvents <- struct{}{}
    c.eventQueue.Close()
    c.lru.Close()
}

// 数据自动同步循环
func (c *Cache) autoSyncLoop() {
    for {
        if r := c.eventQueue.PopFront(); r != nil {
            item := r.(_EventItem)
            newe := c.makeExpireKey(item.e)
            // 查询键名是否已经存在过期时间
            c.emu.RLock()
            olde, ok := c.ekmap[item.k];
            c.emu.RUnlock()
            // 是否需要删除旧的过期时间map中对应的键名
            if ok && newe != olde {
                if ekset := c.getExpireSet(olde); ekset != nil {
                    ekset.Remove(item.k)
                }
            }
            c.getOrNewExpireSet(newe).Add(item.k)
            // 重新设置对应键名的过期时间
            c.emu.Lock()
            c.ekmap[item.k] = newe
            c.emu.Unlock()
            // LRU操作记录(只有新增和修改操作才会记录到LRU管理对象中，删除不会)
            if newe >= olde {
                c.lru.Push(item.k)
            }
        } else {
            return
        }
    }

}

// LRU缓存淘汰处理+自动清理过期键值对
// 每隔1秒清除过去3秒的键值对数据
func (c *Cache) autoClearLoop() {
    for {
        select {
            case <- c.stopEvents:
                return
            default:
                // 缓存过期处理
                ek  := c.makeExpireKey(gtime.Millisecond())
                eks := []int64{ek - 2000, ek - 3000, ek - 4000}
                for _, v := range eks {
                    if ekset := c.getExpireSet(v); ekset != nil {
                        ekset.Iterator(c.clearByKey)
                    }
                    // 删除异步处理键名set
                    c.smu.Lock()
                    delete(c.eksets, v)
                    c.smu.Unlock()
                }
                // LRU缓存淘汰处理
                if c.cap.Val() > 0 {
                    for i := c.Size() - c.cap.Val(); i > 0; i-- {
                        if s := c.lru.Pop(); s != "" {
                            c.clearByKey(s)
                        }
                    }
                }
                time.Sleep(time.Second)
        }
    }
}

// 删除对应键名的缓存数据
func (c *Cache) clearByKey(key string) {
    // 删除缓存数据
    c.dmu.Lock()
    delete(c.data,  key)
    c.dmu.Unlock()

    // 删除异步处理数据项，并更新缓存的内存使用大小记录值
    c.emu.Lock()
    delete(c.ekmap, key)
    c.emu.Unlock()

    // 删除LRU管理对象中指定键名
    c.lru.Remove(key)
}
<|endoftext|>


<commit_before>package generate

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/spf13/cobra"
	"github.com/stretchr/testify/require"
)

func init() {
	runningTests = true
}

func TestGenerateActionArgsComplete(t *testing.T) {
	dir := os.TempDir()
	packagePath := filepath.Join(dir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	r := require.New(t)

	cmd := cobra.Command{}

	e := ActionCmd.RunE(&cmd, []string{})
	r.NotNil(e)

	e = ActionCmd.RunE(&cmd, []string{"users"})
	r.NotNil(e)

	os.Mkdir("actions", 0755)

	e = ActionCmd.RunE(&cmd, []string{"users", "show"})
	r.Nil(e)
}

func TestGenerateActionActionsFolderExists(t *testing.T) {
	dir := os.TempDir()
	packagePath := filepath.Join(dir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	os.RemoveAll("actions")
	os.RemoveAll("templates")

	r := require.New(t)
	cmd := cobra.Command{}

	e := ActionCmd.RunE(&cmd, []string{"users", "show", "edit"})
	r.NotNil(e)

	os.Mkdir("actions", 0755)
	ioutil.WriteFile("actions/app.go", appGo, 0755)

	e = ActionCmd.RunE(&cmd, []string{"users", "show", "edit"})
	r.Nil(e)

	data, _ := ioutil.ReadFile("actions/users.go")
	r.Contains(string(data), "package actions")
	r.Contains(string(data), `import "github.com/gobuffalo/buffalo"`)
	r.Contains(string(data), "func UsersShow(c buffalo.Context) error {")
	r.Contains(string(data), "func UsersEdit(c buffalo.Context) error {")
	r.Contains(string(data), `r.HTML("users/edit.html")`)
	r.Contains(string(data), `c.Render(200, r.HTML("users/show.html"))`)

	data, _ = ioutil.ReadFile("templates/users/show.html")
	r.Contains(string(data), "
Users#Show
")

	data, _ = ioutil.ReadFile("actions/app.go")
	r.Contains(string(data), `app.GET("/users/show", UsersShow)`)
	r.Contains(string(data), `app.GET("/users/edit", UsersEdit)`)
}

func TestGenerateActionActionsFileExists(t *testing.T) {
	dir := os.TempDir()
	packagePath := filepath.Join(dir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	os.Mkdir("actions", 0755)
	ioutil.WriteFile("actions/app.go", appGo, 0755)
	r := require.New(t)
	cmd := cobra.Command{}
	usersContent := `package actions
import "log"

func UsersShow(c buffalo.Context) error {
    log.Println("Something Here!")
    return c.Render(200, r.String("OK"))
}
`
	ioutil.WriteFile("actions/users.go", []byte(usersContent), 0755)

	e := ActionCmd.RunE(&cmd, []string{"users", "show"})
	r.Nil(e)

	data, _ := ioutil.ReadFile("actions/users.go")
	r.Contains(string(data), "log.Println(")
	r.Contains(string(data), "func UsersShow")

}

var appGo = []byte(`
package actions
var app *buffalo.App
func App() *buffalo.App {
	if app == nil {
		app = buffalo.Automatic(buffalo.Options{
			Env: "test",
		})
		app.GET("/", func (c buffalo.Context) error {
			return c.Render(200, r.String("hello"))
		})
	}

	return app
}
`)
<commit_msg>fixed a broken test<commit_after>package generate

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/spf13/cobra"
	"github.com/stretchr/testify/require"
)

func init() {
	runningTests = true
}

func TestGenerateActionArgsComplete(t *testing.T) {
	dir := os.TempDir()
	packagePath := filepath.Join(dir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	r := require.New(t)

	cmd := cobra.Command{}

	e := ActionCmd.RunE(&cmd, []string{})
	r.NotNil(e)

	e = ActionCmd.RunE(&cmd, []string{"users"})
	r.NotNil(e)

	os.Mkdir("actions", 0755)
	ioutil.WriteFile("actions/app.go", appGo, 0755)

	e = ActionCmd.RunE(&cmd, []string{"users", "show"})
	r.Nil(e)
}

func TestGenerateActionActionsFolderExists(t *testing.T) {
	dir := os.TempDir()
	packagePath := filepath.Join(dir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	os.RemoveAll("actions")
	os.RemoveAll("templates")

	r := require.New(t)
	cmd := cobra.Command{}

	e := ActionCmd.RunE(&cmd, []string{"users", "show", "edit"})
	r.NotNil(e)

	os.Mkdir("actions", 0755)
	ioutil.WriteFile("actions/app.go", appGo, 0755)

	e = ActionCmd.RunE(&cmd, []string{"users", "show", "edit"})
	r.Nil(e)

	data, _ := ioutil.ReadFile("actions/users.go")
	r.Contains(string(data), "package actions")
	r.Contains(string(data), `import "github.com/gobuffalo/buffalo"`)
	r.Contains(string(data), "func UsersShow(c buffalo.Context) error {")
	r.Contains(string(data), "func UsersEdit(c buffalo.Context) error {")
	r.Contains(string(data), `r.HTML("users/edit.html")`)
	r.Contains(string(data), `c.Render(200, r.HTML("users/show.html"))`)

	data, _ = ioutil.ReadFile("templates/users/show.html")
	r.Contains(string(data), "
Users#Show
")
}

func TestGenerateActionActionsFileExists(t *testing.T) {
	dir := os.TempDir()
	packagePath := filepath.Join(dir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	os.Mkdir("actions", 0755)
	ioutil.WriteFile("actions/app.go", appGo, 0755)
	r := require.New(t)
	cmd := cobra.Command{}
	usersContent := `package actions
import "log"

func UsersShow(c buffalo.Context) error {
    log.Println("Something Here!")
    return c.Render(200, r.String("OK"))
}
`
	ioutil.WriteFile("actions/users.go", []byte(usersContent), 0755)

	e := ActionCmd.RunE(&cmd, []string{"users", "show"})
	r.Nil(e)

	data, _ := ioutil.ReadFile("actions/users.go")
	r.Contains(string(data), "log.Println(")
	r.Contains(string(data), "func UsersShow")

}

var appGo = []byte(`
package actions
var app *buffalo.App
func App() *buffalo.App {
	if app == nil {
		app = buffalo.Automatic(buffalo.Options{
			Env: "test",
		})
		app.GET("/", func (c buffalo.Context) error {
			return c.Render(200, r.String("hello"))
		})
	}

	return app
}
`)
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	computealpha "google.golang.org/api/compute/v0.alpha"
	compute "google.golang.org/api/compute/v1"
)

func newForwardingRuleMetricContext(request, region string) *metricContext {
	return newForwardingRuleMetricContextWithVersion(request, region, computeV1Version)
}
func newForwardingRuleMetricContextWithVersion(request, region, version string) *metricContext {
	return newGenericMetricContext("forwardingrule", request, region, unusedMetricLabel, version)
}

// CreateGlobalForwardingRule creates the passed GlobalForwardingRule
func (gce *GCECloud) CreateGlobalForwardingRule(rule *compute.ForwardingRule) error {
	mc := newForwardingRuleMetricContext("create", "")
	op, err := gce.service.GlobalForwardingRules.Insert(gce.projectID, rule).Do()
	if err != nil {
		return mc.Observe(err)
	}
	return gce.waitForGlobalOp(op, mc)
}

// SetProxyForGlobalForwardingRule links the given TargetHttp(s)Proxy with the given GlobalForwardingRule.
// targetProxyLink is the SelfLink of a TargetHttp(s)Proxy.
func (gce *GCECloud) SetProxyForGlobalForwardingRule(forwardingRuleName, targetProxyLink string) error {
	mc := newForwardingRuleMetricContext("set_proxy", "")
	op, err := gce.service.GlobalForwardingRules.SetTarget(
		gce.projectID, forwardingRuleName, &compute.TargetReference{Target: targetProxyLink}).Do()
	if err != nil {
		return mc.Observe(err)
	}

	return gce.waitForGlobalOp(op, mc)
}

// DeleteGlobalForwardingRule deletes the GlobalForwardingRule by name.
func (gce *GCECloud) DeleteGlobalForwardingRule(name string) error {
	mc := newForwardingRuleMetricContext("delete", "")
	op, err := gce.service.GlobalForwardingRules.Delete(gce.projectID, name).Do()
	if err != nil {
		return mc.Observe(err)
	}

	return gce.waitForGlobalOp(op, mc)
}

// GetGlobalForwardingRule returns the GlobalForwardingRule by name.
func (gce *GCECloud) GetGlobalForwardingRule(name string) (*compute.ForwardingRule, error) {
	mc := newForwardingRuleMetricContext("get", "")
	v, err := gce.service.GlobalForwardingRules.Get(gce.projectID, name).Do()
	return v, mc.Observe(err)
}

// ListGlobalForwardingRules lists all GlobalForwardingRules in the project.
func (gce *GCECloud) ListGlobalForwardingRules() (*compute.ForwardingRuleList, error) {
	mc := newForwardingRuleMetricContext("list", "")
	// TODO: use PageToken to list all not just the first 500
	v, err := gce.service.GlobalForwardingRules.List(gce.projectID).Do()
	return v, mc.Observe(err)
}

// GetRegionForwardingRule returns the RegionalForwardingRule by name & region.
func (gce *GCECloud) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) {
	mc := newForwardingRuleMetricContext("get", region)
	v, err := gce.service.ForwardingRules.Get(gce.projectID, region, name).Do()
	return v, mc.Observe(err)
}

// GetAlphaRegionForwardingRule returns the Alpha forwarding rule by name & region.
func (gce *GCECloud) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
	mc := newForwardingRuleMetricContextWithVersion("get", region, computeAlphaVersion)
	v, err := gce.serviceAlpha.ForwardingRules.Get(gce.projectID, region, name).Do()
	return v, mc.Observe(err)
}

// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.ForwardingRuleList, error) {
	mc := newForwardingRuleMetricContext("list", region)
	// TODO: use PageToken to list all not just the first 500
	v, err := gce.service.ForwardingRules.List(gce.projectID, region).Do()
	return v, mc.Observe(err)
}

// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (gce *GCECloud) ListAlphaRegionForwardingRules(region string) (*computealpha.ForwardingRuleList, error) {
	mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion)
	// TODO: use PageToken to list all not just the first 500
	v, err := gce.serviceAlpha.ForwardingRules.List(gce.projectID, region).Do()
	return v, mc.Observe(err)
}

// CreateRegionForwardingRule creates and returns a
// RegionalForwardingRule that points to the given BackendService
func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error {
	mc := newForwardingRuleMetricContext("create", region)
	op, err := gce.service.ForwardingRules.Insert(gce.projectID, region, rule).Do()
	if err != nil {
		return mc.Observe(err)
	}

	return gce.waitForRegionOp(op, region, mc)
}

// CreateAlphaRegionForwardingRule creates and returns an Alpha
// forwarding fule in the given region.
func (gce *GCECloud) CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error {
	mc := newForwardingRuleMetricContextWithVersion("create", region, computeAlphaVersion)
	op, err := gce.serviceAlpha.ForwardingRules.Insert(gce.projectID, region, rule).Do()
	if err != nil {
		return mc.Observe(err)
	}

	return gce.waitForRegionOp(op, region, mc)
}

// DeleteRegionForwardingRule deletes the RegionalForwardingRule by name & region.
func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
	mc := newForwardingRuleMetricContext("delete", region)
	op, err := gce.service.ForwardingRules.Delete(gce.projectID, region, name).Do()
	if err != nil {
		return mc.Observe(err)
	}

	return gce.waitForRegionOp(op, region, mc)
}

// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
		return NetworkTierDefault.ToGCEValue(), nil
	}
	fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region)
	if err != nil {
		return handleAlphaNetworkTierGetError(err)
	}
	return fwdRule.NetworkTier, nil
}
<commit_msg>Update Forwarding rules to use generated code<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"context"

	computealpha "google.golang.org/api/compute/v0.alpha"
	compute "google.golang.org/api/compute/v1"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter"
	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
)

func newForwardingRuleMetricContext(request, region string) *metricContext {
	return newForwardingRuleMetricContextWithVersion(request, region, computeV1Version)
}
func newForwardingRuleMetricContextWithVersion(request, region, version string) *metricContext {
	return newGenericMetricContext("forwardingrule", request, region, unusedMetricLabel, version)
}

// CreateGlobalForwardingRule creates the passed GlobalForwardingRule
func (gce *GCECloud) CreateGlobalForwardingRule(rule *compute.ForwardingRule) error {
	mc := newForwardingRuleMetricContext("create", "")
	return mc.Observe(gce.c.GlobalForwardingRules().Insert(context.Background(), meta.GlobalKey(rule.Name), rule))
}

// SetProxyForGlobalForwardingRule links the given TargetHttp(s)Proxy with the given GlobalForwardingRule.
// targetProxyLink is the SelfLink of a TargetHttp(s)Proxy.
func (gce *GCECloud) SetProxyForGlobalForwardingRule(forwardingRuleName, targetProxyLink string) error {
	mc := newForwardingRuleMetricContext("set_proxy", "")
	target := &compute.TargetReference{Target: targetProxyLink}
	return mc.Observe(gce.c.GlobalForwardingRules().SetTarget(context.Background(), meta.GlobalKey(forwardingRuleName), target))
}

// DeleteGlobalForwardingRule deletes the GlobalForwakdingRule by name.
func (gce *GCECloud) DeleteGlobalForwardingRule(name string) error {
	mc := newForwardingRuleMetricContext("delete", "")
	return mc.Observe(gce.c.GlobalForwardingRules().Delete(context.Background(), meta.GlobalKey(name)))
}

// GetGlobalForwardingRule returns the GlobalForwardingRule by name.
func (gce *GCECloud) GetGlobalForwardingRule(name string) (*compute.ForwardingRule, error) {
	mc := newForwardingRuleMetricContext("get", "")
	v, err := gce.c.GlobalForwardingRules().Get(context.Background(), meta.GlobalKey(name))
	return v, mc.Observe(err)
}

// ListGlobalForwardingRules lists all GlobalForwardingRules in the project.
func (gce *GCECloud) ListGlobalForwardingRules() ([]*compute.ForwardingRule, error) {
	mc := newForwardingRuleMetricContext("list", "")
	v, err := gce.c.GlobalForwardingRules().List(context.Background(), filter.None)
	return v, mc.Observe(err)
}

// GetRegionForwardingRule returns the RegionalForwardingRule by name & region.
func (gce *GCECloud) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) {
	mc := newForwardingRuleMetricContext("get", region)
	v, err := gce.c.ForwardingRules().Get(context.Background(), meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}

// GetAlphaRegionForwardingRule returns the Alpha forwarding rule by name & region.
func (gce *GCECloud) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
	mc := newForwardingRuleMetricContextWithVersion("get", region, computeAlphaVersion)
	v, err := gce.c.AlphaForwardingRules().Get(context.Background(), meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}

// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (gce *GCECloud) ListRegionForwardingRules(region string) ([]*compute.ForwardingRule, error) {
	mc := newForwardingRuleMetricContext("list", region)
	v, err := gce.c.ForwardingRules().List(context.Background(), region, filter.None)
	return v, mc.Observe(err)
}

// ListAlphaRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (gce *GCECloud) ListAlphaRegionForwardingRules(region string) ([]*computealpha.ForwardingRule, error) {
	mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion)
	v, err := gce.c.AlphaForwardingRules().List(context.Background(), region, filter.None)
	return v, mc.Observe(err)
}

// CreateRegionForwardingRule creates and returns a
// RegionalForwardingRule that points to the given BackendService
func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error {
	mc := newForwardingRuleMetricContext("create", region)
	return mc.Observe(gce.c.ForwardingRules().Insert(context.Background(), meta.RegionalKey(rule.Name, region), rule))
}

// CreateAlphaRegionForwardingRule creates and returns an Alpha
// forwarding fule in the given region.
func (gce *GCECloud) CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error {
	mc := newForwardingRuleMetricContextWithVersion("create", region, computeAlphaVersion)
	return mc.Observe(gce.c.AlphaForwardingRules().Insert(context.Background(), meta.RegionalKey(rule.Name, region), rule))
}

// DeleteRegionForwardingRule deletes the RegionalForwardingRule by name & region.
func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
	mc := newForwardingRuleMetricContext("delete", region)
	return mc.Observe(gce.c.ForwardingRules().Delete(context.Background(), meta.RegionalKey(name, region)))
}

// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
		return NetworkTierDefault.ToGCEValue(), nil
	}
	fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region)
	if err != nil {
		return handleAlphaNetworkTierGetError(err)
	}
	return fwdRule.NetworkTier, nil
}
<|endoftext|>