<commit_before>package util

import (
	"sync/atomic"
)

// util/ByteBlockPool.java

/*
Class that Posting and PostingVector use to writ ebyte streams into
shared fixed-size []byte arrays. The idea is to allocate slices of
increasing lengths. For example, the first slice is 5 bytes, the next
slice is 14, etc. We start by writing out bytes into the first 5
bytes. When we hit the end of the slice, we allocate the next slice
and then write the address of the next slice into the last 4 bytes of
the previous slice (the "forwarding address").

Each slice is filled with 0's initially, and we mark the end with a
non-zero byte. This way the methods that are writing into the slice
don't need to record its length and instead allocate a new slice once
they hit a non-zero byte.
*/
type ByteBlockPool struct {
	buffers    [][]byte
	bufferUpto int
	byteUpto   int
	buffer     []byte
	byteOffset int
	allocator  ByteAllocator
}

func NewByteBlockPool(allocator ByteAllocator) *ByteBlockPool {
	return &ByteBlockPool{
		bufferUpto: -1,
		byteUpto:   BYTE_BLOCK_SIZE,
		byteOffset: -BYTE_BLOCK_SIZE,
		allocator:  allocator,
	}
}

/* Expert: Resets the pool to its initial state reusing the first buffer. */
func (pool *ByteBlockPool) Reset(zeroFillBuffers, reuseFirst bool) {
	// TODO consolidate with IntBlockPool.Reset()
	if pool.bufferUpto != -1 {
		// We allocated at least one buffer
		if zeroFillBuffers {
			panic("not implemented yet")
		}

		if pool.bufferUpto > 0 || !reuseFirst {
			offset := 0
			if reuseFirst {
				offset = 1
			}
			// Recycle all but the first buffer
			pool.allocator.recycle(pool.buffers[offset : 1+pool.bufferUpto])
			for i := offset; i <= pool.bufferUpto; i++ {
				pool.buffers[i] = nil
			}
		}
		if reuseFirst {
			panic("not implemented yet")
		} else {
			pool.bufferUpto = -1
			pool.byteUpto = BYTE_BLOCK_SIZE
			pool.byteOffset = -BYTE_BLOCK_SIZE
			pool.buffer = nil
		}
	}
}

/*
Advances the pool to its next buffer. This method should be called
once after the constructor to initialize the pool. In contrast to the
constructor, a ByteBlockPool.Reset() call will advance the pool to
its first buffer immediately.
*/
func (pool *ByteBlockPool) NextBuffer() {
	panic("not implemented yet")
}

type ByteAllocator interface {
	recycle(blocks [][]byte)
}

type ByteAllocatorImpl struct {
	blockSize int
}

func newByteAllocator(blockSize int) *ByteAllocatorImpl {
	return &ByteAllocatorImpl{blockSize}
}

/* A simple Allocator that never recycles, but tracks how much total RAM is in use. */
type DirectTrackingAllocator struct {
	*ByteAllocatorImpl
	bytesUsed Counter
}

func NewDirectTrackingAllocator(bytesUsed Counter) *DirectTrackingAllocator {
	return &DirectTrackingAllocator{
		ByteAllocatorImpl: newByteAllocator(BYTE_BLOCK_SIZE),
		bytesUsed:         bytesUsed,
	}
}

func (alloc *DirectTrackingAllocator) recycle(blocks [][]byte) {
	alloc.bytesUsed.AddAndGet(int64(-len(blocks) * alloc.blockSize))
	for i, _ := range blocks {
		blocks[i] = nil
	}
}

// util/Counter.java

type Counter interface {
	AddAndGet(delta int64) int64
	Get() int64
}

func NewCounter() Counter {
	return &serialCounter{0}
}

func NewAtomicCounter() Counter {
	return &atomicCounter{0}
}

type serialCounter struct {
	count int64
}

func (sc *serialCounter) AddAndGet(delta int64) int64 {
	sc.count += delta
	return sc.count
}

func (sc *serialCounter) Get() int64 {
	return sc.count
}

type atomicCounter struct {
	count int64
}

func (ac *atomicCounter) AddAndGet(delta int64) int64 {
	return atomic.AddInt64(&ac.count, delta)
}

func (ac *atomicCounter) Get() int64 {
	return atomic.LoadInt64(&ac.count)
}
<commit_msg>implement ByteBlockPool.nextBuffer()<commit_after>package util

import (
	"sync/atomic"
)

// util/ByteBlockPool.java

/*
Class that Posting and PostingVector use to writ ebyte streams into
shared fixed-size []byte arrays. The idea is to allocate slices of
increasing lengths. For example, the first slice is 5 bytes, the next
slice is 14, etc. We start by writing out bytes into the first 5
bytes. When we hit the end of the slice, we allocate the next slice
and then write the address of the next slice into the last 4 bytes of
the previous slice (the "forwarding address").

Each slice is filled with 0's initially, and we mark the end with a
non-zero byte. This way the methods that are writing into the slice
don't need to record its length and instead allocate a new slice once
they hit a non-zero byte.
*/
type ByteBlockPool struct {
	buffers    [][]byte
	bufferUpto int
	byteUpto   int
	buffer     []byte
	byteOffset int
	allocator  ByteAllocator
}

func NewByteBlockPool(allocator ByteAllocator) *ByteBlockPool {
	return &ByteBlockPool{
		bufferUpto: -1,
		byteUpto:   BYTE_BLOCK_SIZE,
		byteOffset: -BYTE_BLOCK_SIZE,
		allocator:  allocator,
	}
}

/* Expert: Resets the pool to its initial state reusing the first buffer. */
func (pool *ByteBlockPool) Reset(zeroFillBuffers, reuseFirst bool) {
	// TODO consolidate with IntBlockPool.Reset()
	if pool.bufferUpto != -1 {
		// We allocated at least one buffer
		if zeroFillBuffers {
			panic("not implemented yet")
		}

		if pool.bufferUpto > 0 || !reuseFirst {
			offset := 0
			if reuseFirst {
				offset = 1
			}
			// Recycle all but the first buffer
			pool.allocator.recycle(pool.buffers[offset : 1+pool.bufferUpto])
			for i := offset; i <= pool.bufferUpto; i++ {
				pool.buffers[i] = nil
			}
		}
		if reuseFirst {
			panic("not implemented yet")
		} else {
			pool.bufferUpto = -1
			pool.byteUpto = BYTE_BLOCK_SIZE
			pool.byteOffset = -BYTE_BLOCK_SIZE
			pool.buffer = nil
		}
	}
}

/*
Advances the pool to its next buffer. This method should be called
once after the constructor to initialize the pool. In contrast to the
constructor, a ByteBlockPool.Reset() call will advance the pool to
its first buffer immediately.
*/
func (pool *ByteBlockPool) NextBuffer() {
	if 1+pool.bufferUpto == len(pool.buffers) {
		newBuffers := make([][]byte, Oversize(len(pool.buffers)+1, NUM_BYTES_OBJECT_REF))
		copy(newBuffers, pool.buffers)
		pool.buffers = newBuffers
	}
	pool.buffer = pool.allocator.allocate()
	pool.buffers[1+pool.bufferUpto] = pool.buffer
	pool.bufferUpto++

	pool.byteUpto = 0
	pool.byteOffset += BYTE_BLOCK_SIZE
}

/* Abstract class for allocating and freeing byte blocks. */
type ByteAllocator interface {
	recycle(blocks [][]byte)
	allocate() []byte
}

type ByteAllocatorImpl struct {
	blockSize int
}

func newByteAllocator(blockSize int) *ByteAllocatorImpl {
	return &ByteAllocatorImpl{blockSize}
}

func (a *ByteAllocatorImpl) allocate() []byte {
	return make([]byte, a.blockSize)
}

/* A simple Allocator that never recycles, but tracks how much total RAM is in use. */
type DirectTrackingAllocator struct {
	*ByteAllocatorImpl
	bytesUsed Counter
}

func NewDirectTrackingAllocator(bytesUsed Counter) *DirectTrackingAllocator {
	return &DirectTrackingAllocator{
		ByteAllocatorImpl: newByteAllocator(BYTE_BLOCK_SIZE),
		bytesUsed:         bytesUsed,
	}
}

func (alloc *DirectTrackingAllocator) recycle(blocks [][]byte) {
	alloc.bytesUsed.AddAndGet(int64(-len(blocks) * alloc.blockSize))
	for i, _ := range blocks {
		blocks[i] = nil
	}
}

// util/Counter.java

type Counter interface {
	AddAndGet(delta int64) int64
	Get() int64
}

func NewCounter() Counter {
	return &serialCounter{0}
}

func NewAtomicCounter() Counter {
	return &atomicCounter{0}
}

type serialCounter struct {
	count int64
}

func (sc *serialCounter) AddAndGet(delta int64) int64 {
	sc.count += delta
	return sc.count
}

func (sc *serialCounter) Get() int64 {
	return sc.count
}

type atomicCounter struct {
	count int64
}

func (ac *atomicCounter) AddAndGet(delta int64) int64 {
	return atomic.AddInt64(&ac.count, delta)
}

func (ac *atomicCounter) Get() int64 {
	return atomic.LoadInt64(&ac.count)
}
<|endoftext|>


<commit_before>package auth

import (
	"encoding/json"
	"github.com/gorilla/schema"
	"mime"
	"net/http"
)

var formDecoder = schema.NewDecoder()

// TODO support user defined expiration
type Credentials struct {
	UserName string `json:"username" schema:"username"`
	Password string `json:"password" schema:"password"`
}

type LoginResponse struct {
	Token     string    `json:"token"`
	UserID    string    `json:"user_id"`
	UserName  string    `json:"user_name"`
	ExpiredAt Timestamp `json:"expired_at"`
}

func LoginHandler(config *Config) http.Handler {
	config = config.setDefaults()

	fn := func(w http.ResponseWriter, r *http.Request) {
		cred := &Credentials{}
		err := decodePayload(w, r, cred)
		if err != nil {
			return
		}

		user, err := config.UserStore.ValidateCredentials(cred.UserName, cred.Password)
		if err != nil {
			sendError(w, errBadCredentials.cause(err))
			return
		}

		issuedAt := now()
		token := &Token{
			UserID:    user.GetID(),
			UserName:  user.GetName(),
			IssuedAt:  Timestamp(issuedAt),
			ExpiredAt: Timestamp(issuedAt.Add(config.TokenExpiration)),
			ClientIP:  getClientIP(r),
		}

		tokenString, error := token.Encode(config)
		if error != nil {
			sendError(w, error)
			return
		}

		sendJSON(w, &LoginResponse{
			Token:     tokenString,
			UserID:    token.UserID,
			UserName:  token.UserName,
			ExpiredAt: token.ExpiredAt,
		})
	}
	return http.HandlerFunc(fn)
}

func decodePayload(w http.ResponseWriter, r *http.Request, payload interface{}) error {
	contentType := r.Header.Get("Content-Type")
	mediaType, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		sendError(w, errUnsupportedContentType)
		return err
	}

	if mediaType == contentJSON {
		err = json.NewDecoder(r.Body).Decode(payload)
		if err != nil {
			sendError(w, errMalformedContent)
			return err
		}
		return nil
	}

	if mediaType == contentForm {
		err = r.ParseForm()
		if err != nil {
			sendError(w, errUnsupportedContentType)
			return err
		}
		err = formDecoder.Decode(payload, r.PostForm)
		if err != nil {
			sendError(w, errMalformedContent)
			return err
		}
		return nil
	}

	sendError(w, errUnsupportedContentType)
	return errUnsupportedContentType
}
<commit_msg>expose LoginHandlerFunc<commit_after>package auth

import (
	"encoding/json"
	"mime"
	"net/http"

	"github.com/gorilla/schema"
)

var formDecoder = schema.NewDecoder()

// TODO support user defined expiration
type Credentials struct {
	UserName string `json:"username" schema:"username"`
	Password string `json:"password" schema:"password"`
}

type LoginResponse struct {
	Token     string    `json:"token"`
	UserID    string    `json:"user_id"`
	UserName  string    `json:"user_name"`
	ExpiredAt Timestamp `json:"expired_at"`
}

func LoginHandler(config *Config) http.Handler {
	config = config.setDefaults()
	return http.HandlerFunc(LoginHandlerFunc(config))
}

func LoginHandlerFunc(config *Config) http.HandlerFunc {
	config = config.setDefaults()

	return func(w http.ResponseWriter, r *http.Request) {
		cred := &Credentials{}
		err := decodePayload(w, r, cred)
		if err != nil {
			return
		}

		user, err := config.UserStore.ValidateCredentials(cred.UserName, cred.Password)
		if err != nil {
			sendError(w, errBadCredentials.cause(err))
			return
		}

		issuedAt := now()
		token := &Token{
			UserID:    user.GetID(),
			UserName:  user.GetName(),
			IssuedAt:  Timestamp(issuedAt),
			ExpiredAt: Timestamp(issuedAt.Add(config.TokenExpiration)),
			ClientIP:  getClientIP(r),
		}

		tokenString, error := token.Encode(config)
		if error != nil {
			sendError(w, error)
			return
		}

		sendJSON(w, &LoginResponse{
			Token:     tokenString,
			UserID:    token.UserID,
			UserName:  token.UserName,
			ExpiredAt: token.ExpiredAt,
		})
	}
}

func decodePayload(w http.ResponseWriter, r *http.Request, payload interface{}) error {
	contentType := r.Header.Get("Content-Type")
	mediaType, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		sendError(w, errUnsupportedContentType)
		return err
	}

	if mediaType == contentJSON {
		err = json.NewDecoder(r.Body).Decode(payload)
		if err != nil {
			sendError(w, errMalformedContent)
			return err
		}
		return nil
	}

	if mediaType == contentForm {
		err = r.ParseForm()
		if err != nil {
			sendError(w, errUnsupportedContentType)
			return err
		}
		err = formDecoder.Decode(payload, r.PostForm)
		if err != nil {
			sendError(w, errMalformedContent)
			return err
		}
		return nil
	}

	sendError(w, errUnsupportedContentType)
	return errUnsupportedContentType
}
<|endoftext|>


<commit_before>/*
Copyright 2017 wechat-go Authors. All Rights Reserved.
MIT License

Copyright (c) 2017

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package wxbot

import (
	"fmt"
	"github.com/mdp/qrterminal"
	"github.com/songtianyi/rrframework/config"
	//"github.com/songtianyi/rrframework/storage"
	"github.com/songtianyi/rrframework/logs"
	"github.com/songtianyi/wechat-go/wxweb"
	"net/http"
	"os"
	"time"
)

var (
	WxWebDefaultCommon *wxweb.Common
	WxWebXcg           *wxweb.XmlConfig
	Cookies            []*http.Cookie
	SynKeyList         *wxweb.SyncKeyList
	Bot                *wxweb.User
	Cm *ContactManager
)

func init() {
	WxWebDefaultCommon = &wxweb.Common{
		AppId:     "wx782c26e4c19acffb",
		LoginUrl:  "https://login.weixin.qq.com",
		Lang:      "zh_CN",
		DeviceID:  "e" + wxweb.GetRandomStringFromNum(15),
		UserAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36",
		CgiUrl:    "https://wx.qq.com/cgi-bin/mmwebwx-bin",
		SyncSrvs: []string{
			"webpush.wx.qq.com",
			"webpush.weixin.qq.com",
			"webpush.wechat.com",
			"webpush1.wechat.com",
			"webpush2.wechat.com",
		},
		UploadUrl: "https://file.wx.qq.com/cgi-bin/mmwebwx-bin/webwxuploadmedia?f=json",
		MediaCount: 0,
		RedirectUri: "https://wx.qq.com/cgi-bin/mmwebwx-bin/webwxnewloginpage",
	}
	WxWebXcg = &wxweb.XmlConfig{}
}

func AutoLogin() {
	logs.Debug("%v", WxWebDefaultCommon)
	uuid, err := wxweb.JsLogin(WxWebDefaultCommon)
	if err != nil {
		panic(err)
	}
	logs.Debug(uuid)
	qrterminal.Generate("https://login.weixin.qq.com/l/"+uuid, qrterminal.L, os.Stdout)
	f, err := os.Create("qrcoe.jpg")
	if err != nil {
		panic(err)
	}
	qrterminal.Generate("https://login.weixin.qq.com/l/"+uuid, qrterminal.L, f)
	f.Close()

	//qrcb, err := wxweb.QrCode(WxWebDefaultCommon, uuid)
	//if err != nil {
	//	panic(err)
	//}
	//ls := rrstorage.CreateLocalDiskStorage("./")
	//if err := ls.Save(qrcb, "qrcode.jpg"); err != nil {
	//	panic(err)
	//}

	redirectUri := ""
loop1:
	for {
		select {
		case <-time.After(5 * time.Second):
			redirectUri, err = wxweb.Login(WxWebDefaultCommon, uuid, "0")
			if err != nil {
				logs.Error(err)
			} else {
				break loop1
			}
		}
	}
	logs.Debug(redirectUri)

	if Cookies, err = wxweb.WebNewLoginPage(WxWebDefaultCommon, WxWebXcg, redirectUri); err != nil {
		panic(err)
	}

	jb, err := wxweb.WebWxInit(WxWebDefaultCommon, WxWebXcg)
	if err != nil {
		panic(err)
	}

	jc, err := rrconfig.LoadJsonConfigFromBytes(jb)
	if err != nil {
		panic(err)
	}

	SynKeyList, err = wxweb.GetSyncKeyListFromJc(jc)
	if err != nil {
		panic(err)
	}
	Bot, _ = wxweb.GetUserInfoFromJc(jc)
	logs.Debug(Bot)
	ret, err := wxweb.WebWxStatusNotify(WxWebDefaultCommon, WxWebXcg, Bot)
	if err != nil {
		panic(err)
	}
	if ret != 0 {
		panic(fmt.Errorf("WebWxStatusNotify fail, %d", ret))
	}

	cb, err := wxweb.WebWxGetContact(WxWebDefaultCommon, WxWebXcg, Cookies)
	if err != nil {
		panic(err)
	}
	Cm, err = LoadContactFromBytes(cb)
	if err != nil {
		panic(err)
	}

	_, err = wxweb.WebWxBatchGetContact(WxWebDefaultCommon, WxWebXcg, Cookies, Cm.GetGroupContact())
	if err != nil {
		panic(err)
	}
}
<commit_msg>try fix<commit_after>/*
Copyright 2017 wechat-go Authors. All Rights Reserved.
MIT License

Copyright (c) 2017

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package wxbot

import (
	"fmt"
	//"github.com/mdp/qrterminal"
	"github.com/songtianyi/rrframework/config"
	"github.com/songtianyi/rrframework/storage"
	"github.com/songtianyi/rrframework/logs"
	"github.com/songtianyi/wechat-go/wxweb"
	"net/http"
	//"os"
	"time"
)

var (
	WxWebDefaultCommon *wxweb.Common
	WxWebXcg           *wxweb.XmlConfig
	Cookies            []*http.Cookie
	SynKeyList         *wxweb.SyncKeyList
	Bot                *wxweb.User
	Cm *ContactManager
)

func init() {
	WxWebDefaultCommon = &wxweb.Common{
		AppId:     "wx782c26e4c19acffb",
		LoginUrl:  "https://login.weixin.qq.com",
		Lang:      "zh_CN",
		DeviceID:  "e" + wxweb.GetRandomStringFromNum(15),
		UserAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36",
		CgiUrl:    "https://wx.qq.com/cgi-bin/mmwebwx-bin",
		SyncSrvs: []string{
			"webpush.wx.qq.com",
			"webpush.weixin.qq.com",
			"webpush.wechat.com",
			"webpush1.wechat.com",
			"webpush2.wechat.com",
		},
		UploadUrl: "https://file.wx.qq.com/cgi-bin/mmwebwx-bin/webwxuploadmedia?f=json",
		MediaCount: 0,
		RedirectUri: "https://wx.qq.com/cgi-bin/mmwebwx-bin/webwxnewloginpage",
	}
	WxWebXcg = &wxweb.XmlConfig{}
}

func AutoLogin() {
	logs.Debug("%v", WxWebDefaultCommon)
	uuid, err := wxweb.JsLogin(WxWebDefaultCommon)
	if err != nil {
		panic(err)
	}
	logs.Debug(uuid)
	//qrterminal.Generate("https://login.weixin.qq.com/l/"+uuid, qrterminal.L, os.Stdout)

	qrcb, err := wxweb.QrCode(WxWebDefaultCommon, uuid)
	if err != nil {
		panic(err)
	}
	ls := rrstorage.CreateLocalDiskStorage("./")
	if err := ls.Save(qrcb, "qrcode.jpg"); err != nil {
		panic(err)
	}

	redirectUri := ""
loop1:
	for {
		select {
		case <-time.After(5 * time.Second):
			redirectUri, err = wxweb.Login(WxWebDefaultCommon, uuid, "0")
			if err != nil {
				logs.Error(err)
			} else {
				break loop1
			}
		}
	}
	logs.Debug(redirectUri)

	if Cookies, err = wxweb.WebNewLoginPage(WxWebDefaultCommon, WxWebXcg, redirectUri); err != nil {
		panic(err)
	}

	jb, err := wxweb.WebWxInit(WxWebDefaultCommon, WxWebXcg)
	if err != nil {
		panic(err)
	}

	jc, err := rrconfig.LoadJsonConfigFromBytes(jb)
	if err != nil {
		panic(err)
	}

	SynKeyList, err = wxweb.GetSyncKeyListFromJc(jc)
	if err != nil {
		panic(err)
	}
	Bot, _ = wxweb.GetUserInfoFromJc(jc)
	logs.Debug(Bot)
	ret, err := wxweb.WebWxStatusNotify(WxWebDefaultCommon, WxWebXcg, Bot)
	if err != nil {
		panic(err)
	}
	if ret != 0 {
		panic(fmt.Errorf("WebWxStatusNotify fail, %d", ret))
	}

	cb, err := wxweb.WebWxGetContact(WxWebDefaultCommon, WxWebXcg, Cookies)
	if err != nil {
		panic(err)
	}
	Cm, err = LoadContactFromBytes(cb)
	if err != nil {
		panic(err)
	}

	_, err = wxweb.WebWxBatchGetContact(WxWebDefaultCommon, WxWebXcg, Cookies, Cm.GetGroupContact())
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package crypto

import (
	"encoding/json"
	"sort"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNewContractAddress(t *testing.T) {
	addr := NewContractAddress(Address{
		233, 181, 216, 115, 19,
		53, 100, 101, 250, 227,
		60, 64, 108, 226, 194,
		151, 157, 230, 11, 203,
	}, []byte{1})

	assert.Equal(t, Address{
		0xe2, 0x9d, 0x64, 0x7d, 0xce, 0xd6, 0x48, 0x12, 0xd4, 0x44,
		0xa6, 0x7d, 0x80, 0x5, 0xcd, 0x6, 0x9, 0xb2, 0x2f, 0x49,
	}, addr)
}

func TestNewContractAddress2(t *testing.T) {
	addr := NewContractAddress2(Address{
		0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,
	}, 1, [32]byte{
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
	}, []byte{0x00})

	assert.Equal(t, Address{
		77, 26, 46, 43, 180,
		248, 143, 2, 80, 242,
		111, 255, 240, 152, 176,
		179, 11, 38, 191, 56,
	}, addr)
}

func TestAddress_MarshalJSON(t *testing.T) {
	addr := Address{
		73, 234, 48, 252, 174,
		115, 27, 222, 54, 116,
		47, 133, 144, 21, 73,
		245, 21, 234, 26, 50,
	}

	bs, err := json.Marshal(addr)
	assert.NoError(t, err)

	addrOut := new(Address)
	err = json.Unmarshal(bs, addrOut)

	assert.Equal(t, addr, *addrOut)
}

func TestAddress_MarshalText(t *testing.T) {
	addr := Address{
		73, 234, 48, 252, 174,
		115, 27, 222, 54, 116,
		47, 133, 144, 21, 73,
		245, 21, 234, 26, 50,
	}

	bs, err := addr.MarshalText()
	assert.NoError(t, err)

	addrOut := new(Address)
	err = addrOut.UnmarshalText(bs)

	assert.Equal(t, addr, *addrOut)
}

func TestAddress_Length(t *testing.T) {
	addrOut := new(Address)
	err := addrOut.UnmarshalText(([]byte)("49EA30FCAE731BDE36742F85901549F515EA1A10"))
	require.NoError(t, err)

	err = addrOut.UnmarshalText(([]byte)("49EA30FCAE731BDE36742F85901549F515EA1A1"))
	assert.Error(t, err, "address too short")

	err = addrOut.UnmarshalText(([]byte)("49EA30FCAE731BDE36742F85901549F515EA1A1020"))
	assert.Error(t, err, "address too long")
}

func TestAddress_Sort(t *testing.T) {
	addresses := Addresses{
		{2, 3, 4},
		{3, 1, 2},
		{2, 1, 2},
	}
	sorted := make(Addresses, len(addresses))
	copy(sorted, addresses)
	sort.Stable(sorted)
	assert.Equal(t, Addresses{
		{2, 1, 2},
		{2, 3, 4},
		{3, 1, 2},
	}, sorted)
}
<commit_msg>fix crypto test<commit_after>package crypto

import (
	"encoding/json"
	"sort"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNewContractAddress(t *testing.T) {
	addr := NewContractAddress(Address{
		233, 181, 216, 115, 19,
		53, 100, 101, 250, 227,
		60, 64, 108, 226, 194,
		151, 157, 230, 11, 203,
	}, []byte{1})

	assert.Equal(t, Address{
		0xe2, 0x9d, 0x64, 0x7d, 0xce, 0xd6, 0x48, 0x12, 0xd4, 0x44,
		0xa6, 0x7d, 0x80, 0x5, 0xcd, 0x6, 0x9, 0xb2, 0x2f, 0x49,
	}, addr)
}

func TestNewContractAddress2(t *testing.T) {
	addr := NewContractAddress2(Address{
		0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,
	}, []byte{1}, [32]byte{
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
	}, []byte{0x00})

	assert.Equal(t, Address{
		77, 26, 46, 43, 180,
		248, 143, 2, 80, 242,
		111, 255, 240, 152, 176,
		179, 11, 38, 191, 56,
	}, addr)
}

func TestAddress_MarshalJSON(t *testing.T) {
	addr := Address{
		73, 234, 48, 252, 174,
		115, 27, 222, 54, 116,
		47, 133, 144, 21, 73,
		245, 21, 234, 26, 50,
	}

	bs, err := json.Marshal(addr)
	assert.NoError(t, err)

	addrOut := new(Address)
	err = json.Unmarshal(bs, addrOut)

	assert.Equal(t, addr, *addrOut)
}

func TestAddress_MarshalText(t *testing.T) {
	addr := Address{
		73, 234, 48, 252, 174,
		115, 27, 222, 54, 116,
		47, 133, 144, 21, 73,
		245, 21, 234, 26, 50,
	}

	bs, err := addr.MarshalText()
	assert.NoError(t, err)

	addrOut := new(Address)
	err = addrOut.UnmarshalText(bs)

	assert.Equal(t, addr, *addrOut)
}

func TestAddress_Length(t *testing.T) {
	addrOut := new(Address)
	err := addrOut.UnmarshalText(([]byte)("49EA30FCAE731BDE36742F85901549F515EA1A10"))
	require.NoError(t, err)

	err = addrOut.UnmarshalText(([]byte)("49EA30FCAE731BDE36742F85901549F515EA1A1"))
	assert.Error(t, err, "address too short")

	err = addrOut.UnmarshalText(([]byte)("49EA30FCAE731BDE36742F85901549F515EA1A1020"))
	assert.Error(t, err, "address too long")
}

func TestAddress_Sort(t *testing.T) {
	addresses := Addresses{
		{2, 3, 4},
		{3, 1, 2},
		{2, 1, 2},
	}
	sorted := make(Addresses, len(addresses))
	copy(sorted, addresses)
	sort.Stable(sorted)
	assert.Equal(t, Addresses{
		{2, 1, 2},
		{2, 3, 4},
		{3, 1, 2},
	}, sorted)
}
<|endoftext|>


<commit_before>package poloniex

import (
	"encoding/json"
	"time"
)

const (
	TRADE_FILL_OR_KILL        = "fillOrKill"
	TRADE_IMMEDIATE_OR_CANCEL = "immediateOrCancel"
	TRADE_POST_ONLY           = "postOnly"
)

type Trade struct {
	GlobalTradeID uint64    `json:"globalTradeID"`
	TradeID       uint64    `json:"tradeID,string"`
	Date          time.Time `json:"date,string"`
	Type          string    `json:"type"`
	Category      string    `json:"category"`
	Rate          float64   `json:"rate,string"`
	Amount        float64   `json:"amount,string"`
	Total         float64   `json:"total,string"`
	Fee           float64   `json:"fee,string"`
}

func (t *Trade) UnmarshalJSON(data []byte) error {
	var err error
	type Alias Trade
	aux := &struct {
		Date string `json:"date"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Date, err = time.Parse("2006-01-02 15:04:05", aux.Date)
	if err != nil {
		return err
	}
	return nil
}

type ResultingTrade struct {
	Amount  float64 `json:"amount"`
	Date    string  `json:"date"`
	Rate    float64 `json:"rate"`
	Total   float64 `json:"total"`
	TradeID string  `json:"tradeID"`
	Type    string  `json:"type"`
}

type TradeOrder struct {
	OrderNumber     string           `json:"orderNumber"`
	ResultingTrades []ResultingTrade `json:"resultingTrades"`
}
<commit_msg>type conversion during trade unmarshalling<commit_after>package poloniex

import (
	"encoding/json"
	"time"
)

const (
	TRADE_FILL_OR_KILL        = "fillOrKill"
	TRADE_IMMEDIATE_OR_CANCEL = "immediateOrCancel"
	TRADE_POST_ONLY           = "postOnly"
)

type Trade struct {
	GlobalTradeID uint64    `json:"globalTradeID"`
	TradeID       uint64    `json:"tradeID,string"`
	Date          time.Time `json:"date,string"`
	Type          string    `json:"type"`
	Category      string    `json:"category"`
	Rate          float64   `json:"rate,string"`
	Amount        float64   `json:"amount,string"`
	Total         float64   `json:"total,string"`
	Fee           float64   `json:"fee,string"`
}

func (t *Trade) UnmarshalJSON(data []byte) error {
	var err error
	type Alias Trade
	aux := &struct {
		Date string `json:"date"`
		*Alias
	}{
		Alias: (*Alias)(t),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}
	t.Date, err = time.Parse("2006-01-02 15:04:05", aux.Date)
	if err != nil {
		return err
	}
	return nil
}

type ResultingTrade struct {
	Amount  float64 `json:"amount,string"`
	Date    string  `json:"date"`
	Rate    float64 `json:"rate,string"`
	Total   float64 `json:"total,string"`
	TradeID string  `json:"tradeID"`
	Type    string  `json:"type"`
}

type TradeOrder struct {
	OrderNumber     string           `json:"orderNumber"`
	ResultingTrades []ResultingTrade `json:"resultingTrades"`
}
<|endoftext|>


<commit_before>package lib

import (
	"io"
	"os"
	"sort"
	"sync"
)

type Source interface {
	Open() (Reader, error)
}

type SourceFunc func() (Reader, error)

func (f SourceFunc) Open() (Reader, error) {
	return f()
}

func RegisterSource(name string, source Source) {
	srcmtx.Lock()
	srcmap[name] = source
	srcmtx.Unlock()
}

func DeregisterSource(name string) {
	srcmtx.Lock()
	delete(srcmap, name)
	srcmtx.Unlock()
}

func GetSource(name string) (source Source) {
	srcmtx.RLock()
	source = srcmap[name]
	srcmtx.RUnlock()
	return
}

func GetSources(names ...string) (sources []Source) {
	sources = make([]Source, 0, len(names))

	for _, name := range names {
		if source := GetSource(name); source != nil {
			sources = append(sources, source)
		}
	}

	return
}

func SourcesAvailable() (sources []string) {
	srcmtx.RLock()
	sources = make([]string, 0, len(srcmap))

	for name := range srcmap {
		sources = append(sources, name)
	}

	srcmtx.RUnlock()
	sort.Strings(sources)
	return
}

var (
	srcmtx sync.RWMutex
	srcmap = map[string]Source{
		"stdin": SourceFunc(func() (Reader, error) {
			// On some platforms closing stdin doesn't cause pending read
			// operations to abort, resulting in a blocking call that never
			// returns.
			//
			// To work around this limitation we start a goroutine that is
			// responsible for reading from stdin and sending the bytes through
			// an in-memory pipe. When the pipe is closed it properly cancels
			// all pending reads which is the behavior we expect.
			//
			// There probably is a small performance cost to adding this extra
			// step but the stdin source shouldn't be used in production
			// environments so it shouldn't be a problem in practice.
			//
			// Note that the goroutine reading from stdin is likely gonna be
			// leaked... This is OK in the ecs-logs use case because only one
			// stdin reader will be instantiated.
			r, w := io.Pipe()

			go func() {
				defer w.Close()
				io.Copy(w, os.Stdin)
			}()

			// We use the Close method of the write end of the pipe so when it's
			// called the read end will start returning io.EOF to indicate a
			// graceful shutdown.
			return NewMessageDecoder(struct {
				io.Reader
				io.Closer
			}{r, w}), nil
		}),
	}
)
<commit_msg>[hot fix] forward the actual error from stdin through the pipe<commit_after>package lib

import (
	"io"
	"os"
	"sort"
	"sync"
)

type Source interface {
	Open() (Reader, error)
}

type SourceFunc func() (Reader, error)

func (f SourceFunc) Open() (Reader, error) {
	return f()
}

func RegisterSource(name string, source Source) {
	srcmtx.Lock()
	srcmap[name] = source
	srcmtx.Unlock()
}

func DeregisterSource(name string) {
	srcmtx.Lock()
	delete(srcmap, name)
	srcmtx.Unlock()
}

func GetSource(name string) (source Source) {
	srcmtx.RLock()
	source = srcmap[name]
	srcmtx.RUnlock()
	return
}

func GetSources(names ...string) (sources []Source) {
	sources = make([]Source, 0, len(names))

	for _, name := range names {
		if source := GetSource(name); source != nil {
			sources = append(sources, source)
		}
	}

	return
}

func SourcesAvailable() (sources []string) {
	srcmtx.RLock()
	sources = make([]string, 0, len(srcmap))

	for name := range srcmap {
		sources = append(sources, name)
	}

	srcmtx.RUnlock()
	sort.Strings(sources)
	return
}

var (
	srcmtx sync.RWMutex
	srcmap = map[string]Source{
		"stdin": SourceFunc(func() (Reader, error) {
			// On some platforms closing stdin doesn't cause pending read
			// operations to abort, resulting in a blocking call that never
			// returns.
			//
			// To work around this limitation we start a goroutine that is
			// responsible for reading from stdin and sending the bytes through
			// an in-memory pipe. When the pipe is closed it properly cancels
			// all pending reads which is the behavior we expect.
			//
			// There probably is a small performance cost to adding this extra
			// step but the stdin source shouldn't be used in production
			// environments so it shouldn't be a problem in practice.
			//
			// Note that the goroutine reading from stdin is likely gonna be
			// leaked... This is OK in the ecs-logs use case because only one
			// stdin reader will be instantiated.
			r, w := io.Pipe()
			go pipe(w, os.Stdin)
			// We use the Close method of the write end of the pipe so when it's
			// called the read end will start returning io.EOF to indicate a
			// graceful shutdown.
			return NewMessageDecoder(struct {
				io.Reader
				io.Closer
			}{r, w}), nil
		}),
	}
)

func pipe(w *io.PipeWriter, r io.Reader) {
	_, err := io.Copy(w, r)

	if err == nil {
		err = io.EOF
	}

	w.CloseWithError(err)
}
<|endoftext|>


<commit_before>package gdoc

import (
	"testing"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

func makeBlinkNode() *html.Node {
	return &html.Node{
		Type:     html.ElementNode,
		DataAtom: atom.Blink,
		Data:     "blink",
	}
}

func makePNode() *html.Node {
	return &html.Node{
		Type:     html.ElementNode,
		DataAtom: atom.P,
		Data:     "p",
	}
}

// data is the text in the node.
func makeTextNode(data string) *html.Node {
	return &html.Node{
		Type: html.TextNode,
		Data: data,
	}
}

func TestIsHeader(t *testing.T) {
	tests := []struct {
		name string
		in   *html.Node
		out  bool
	}{
		{
			name: "StepTitle",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H1,
				Data:     "h1",
			},
		},
		{
			name: "FirstLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H2,
				Data:     "h2",
			},
			out: true,
		},
		{
			name: "SecondLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H3,
				Data:     "h3",
			},
			out: true,
		},
		{
			name: "ThirdLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H4,
				Data:     "h4",
			},
			out: true,
		},
		{
			name: "FourthLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H5,
				Data:     "h5",
			},
			out: true,
		},
		{
			name: "FifthLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H6,
				Data:     "h6",
			},
			out: true,
		},
		{
			name: "NotAHeader",
			in:   makeBlinkNode(),
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			if out := isHeader(tc.in); out != tc.out {
				t.Errorf("isHeader(%v) = %t, want %t", tc.in, out, tc.out)
			}
		})
	}
}

// TODO: test isMeta

// TODO: test isBold

// TODO: test isItalic

// TODO: test isConsole

// TODO: test isCode

// TODO: test isButton

// TODO: test isInfobox

// TODO: test isInfoboxNegative

// TODO: test isSurvey

// TODO: test isComment

// TODO: test isTable

// TODO: test isList

// TODO: test countTwo

func TestCountDirect(t *testing.T) {
	a1 := makePNode()
	a2 := makeTextNode("foobar")
	a1.AppendChild(a2)

	b1 := makePNode()
	b2 := makeTextNode("foobar")
	b3 := makeTextNode("foobar2")
	b4 := makeTextNode("foobar3")
	// The nodes should be siblings.
	b1.AppendChild(b2)
	b1.AppendChild(b3)
	b1.AppendChild(b4)

	c1 := makePNode()
	c2 := makeBlinkNode()
	c3 := makeTextNode("foobar")
	c1.AppendChild(c2)
	c2.AppendChild(c3)

	tests := []struct {
		name string
		in   *html.Node
		out  int
	}{
		{
			name: "Zero",
			in:   makePNode(),
			out:  0,
		},
		{
			name: "One",
			in:   a1,
			out:  1,
		},
		{
			name: "MoreThanOne",
			in:   b1,
			out:  3,
		},
		{
			name: "NonRecursive",
			in:   c1,
			out:  1,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			if out := countDirect(tc.in); out != tc.out {
				t.Errorf("countDirect(%+v) = %d, want %d", tc.in, out, tc.out)
			}
		})
	}
}

// TODO: test findAtom

// TODO: test findChildAtoms

// TODO: test findParent

// TODO: test findBlockParent

// TODO: test nodeAttr

// TODO: test stringifyNode
<commit_msg>Add tests for gdoc's nodeAttr.<commit_after>package gdoc

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

func makeBlinkNode() *html.Node {
	return &html.Node{
		Type:     html.ElementNode,
		DataAtom: atom.Blink,
		Data:     "blink",
	}
}

func makePNode() *html.Node {
	return &html.Node{
		Type:     html.ElementNode,
		DataAtom: atom.P,
		Data:     "p",
	}
}

// data is the text in the node.
func makeTextNode(data string) *html.Node {
	return &html.Node{
		Type: html.TextNode,
		Data: data,
	}
}

func TestIsHeader(t *testing.T) {
	tests := []struct {
		name string
		in   *html.Node
		out  bool
	}{
		{
			name: "StepTitle",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H1,
				Data:     "h1",
			},
		},
		{
			name: "FirstLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H2,
				Data:     "h2",
			},
			out: true,
		},
		{
			name: "SecondLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H3,
				Data:     "h3",
			},
			out: true,
		},
		{
			name: "ThirdLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H4,
				Data:     "h4",
			},
			out: true,
		},
		{
			name: "FourthLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H5,
				Data:     "h5",
			},
			out: true,
		},
		{
			name: "FifthLevel",
			in: &html.Node{
				Type:     html.ElementNode,
				DataAtom: atom.H6,
				Data:     "h6",
			},
			out: true,
		},
		{
			name: "NotAHeader",
			in:   makeBlinkNode(),
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			if out := isHeader(tc.in); out != tc.out {
				t.Errorf("isHeader(%v) = %t, want %t", tc.in, out, tc.out)
			}
		})
	}
}

// TODO: test isMeta

// TODO: test isBold

// TODO: test isItalic

// TODO: test isConsole

// TODO: test isCode

// TODO: test isButton

// TODO: test isInfobox

// TODO: test isInfoboxNegative

// TODO: test isSurvey

// TODO: test isComment

// TODO: test isTable

// TODO: test isList

// TODO: test countTwo

func TestCountDirect(t *testing.T) {
	a1 := makePNode()
	a2 := makeTextNode("foobar")
	a1.AppendChild(a2)

	b1 := makePNode()
	b2 := makeTextNode("foobar")
	b3 := makeTextNode("foobar2")
	b4 := makeTextNode("foobar3")
	// The nodes should be siblings.
	b1.AppendChild(b2)
	b1.AppendChild(b3)
	b1.AppendChild(b4)

	c1 := makePNode()
	c2 := makeBlinkNode()
	c3 := makeTextNode("foobar")
	c1.AppendChild(c2)
	c2.AppendChild(c3)

	tests := []struct {
		name string
		in   *html.Node
		out  int
	}{
		{
			name: "Zero",
			in:   makePNode(),
			out:  0,
		},
		{
			name: "One",
			in:   a1,
			out:  1,
		},
		{
			name: "MoreThanOne",
			in:   b1,
			out:  3,
		},
		{
			name: "NonRecursive",
			in:   c1,
			out:  1,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			if out := countDirect(tc.in); out != tc.out {
				t.Errorf("countDirect(%+v) = %d, want %d", tc.in, out, tc.out)
			}
		})
	}
}

// TODO: test findAtom

// TODO: test findChildAtoms

// TODO: test findParent

// TODO: test findBlockParent

func TestNodeAttr(t *testing.T) {
	a1 := makeBlinkNode()
	a1.Attr = append(a1.Attr, html.Attribute{Key: "keyone", Val: "valone"})
	a1.Attr = append(a1.Attr, html.Attribute{Key: "keytwo", Val: "valtwo"})
	a1.Attr = append(a1.Attr, html.Attribute{Key: "keythree", Val: "valthree"})

	tests := []struct {
		name   string
		inNode *html.Node
		inKey  string
		out    string
	}{
		{
			name:   "Simple",
			inNode: a1,
			inKey:  "keyone",
			out:    "valone",
		},
		{
			name:   "MixedCase",
			inNode: a1,
			inKey:  "KEytWO",
			out:    "valtwo",
		},
		{
			name:   "NotFound",
			inNode: a1,
			inKey:  "nokey",
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			if diff := cmp.Diff(tc.out, nodeAttr(tc.inNode, tc.inKey)); diff != "" {
				t.Errorf("nodeAttr(%+v, %s) got diff (-want +got):\n%s", tc.inNode, tc.inKey, diff)
			}
		})
	}
}

// TODO: test stringifyNode
<|endoftext|>


<commit_before>package cloudformation

import (
	"fmt"
	"log"
	"os"
	"sort"
	"time"

	"github.com/dynport/dgtk/cli"
	"github.com/dynport/gocli"
	"github.com/dynport/gocloud/aws/cloudformation"
)

var logger = log.New(os.Stdout, "", 0)

type StacksDelete struct {
	Name string `cli:"arg required"`
}

func (d *StacksDelete) Run() error {
	logger.Printf("deleting stack %q", d.Name)
	return client.DeleteStack(d.Name)
}

type StacksList struct {
	IncludeDeleted bool `cli:"opt --deleted"`
	Full           bool `cli:"opt --full"`
}

var client = cloudformation.NewFromEnv()

func (list *StacksList) Run() error {
	rsp, e := client.ListStacks(nil)
	if e != nil {
		return e
	}
	t := gocli.NewTable()
	for _, s := range rsp.ListStacksResult.Stacks {
		if !list.IncludeDeleted && s.StackStatus == "DELETE_COMPLETE" {
			continue
		}
		parts := []interface{}{s.StackName, s.StackStatus, s.CreationTime.Format(time.RFC3339)}
		if list.Full {
			parts = append(parts, s.StackId)
		}
		t.Add(parts...)
	}
	fmt.Println(t)
	return nil
}

type StackResources struct {
	Name string `cli:"arg required"`
}

func (r *StackResources) Run() error {
	rsp, e := client.DescribeStackResources(cloudformation.DescribeStackResourcesParameters{
		StackName: r.Name,
	})
	if e != nil {
		return e
	}
	t := gocli.NewTable()
	for _, r := range rsp.DescribeStackResourcesResult.StackResources {
		t.Add(r.LogicalResourceId, r.PhysicalResourceId)
	}
	fmt.Println(t)
	return nil
}

type StackEventsList []*cloudformation.StackEvent

func (list StackEventsList) Len() int {
	return len(list)
}

func (list StackEventsList) Swap(a, b int) {
	list[a], list[b] = list[b], list[a]
}

func (list StackEventsList) Less(a, b int) bool {
	return list[a].Timestamp.Before(list[b].Timestamp)
}

var (
	ReasonUserInitiated             = "User Initiated"
	ReasonResourceCreationInitiated = "Resource creation Initiated"
)

type StacksWatch struct {
	Name    string `cli:"arg required"`
	Refresh int    `cli:"opt --refresh default=5"`
}

func (s *StacksWatch) Run() error {
	last := time.Time{}
	for {
		rsp, e := client.DescribeStackEvents(&cloudformation.DescribeStackEventsParameters{StackName: s.Name})
		if e != nil {
			fmt.Printf("ERROR: %s\n", e.Error())
		} else {
			events := StackEventsList(rsp.DescribeStackEventsResult.StackEvents)
			sort.Sort(events)
			for _, e := range events {
				if e.Timestamp.After(last) {
					ph := e.PhysicalResourceId
					fmt.Printf("%s %-24s %-32s %s\n", e.Timestamp.Format(time.RFC3339), maxLen(e.LogicalResourceId, 24), maxLen(ph, 32), e.ResourceStatus)
					switch e.ResourceStatusReason {
					case "", ReasonUserInitiated, ReasonResourceCreationInitiated:
						//
					default:
						fmt.Printf("%20s %s\n", "", gocli.Red(e.ResourceStatusReason))
					}
					last = e.Timestamp
				}
			}
		}
		time.Sleep(time.Duration(s.Refresh) * time.Second)
	}
	return nil
}

func maxLen(s string, i int) string {
	if len(s) < i {
		return s
	}
	return s[0:i]
}

func Register(router *cli.Router) {
	router.Register("aws/cloudformation/stacks/delete", &StacksDelete{}, "Delete Stack")
	router.Register("aws/cloudformation/stacks/list", &StacksList{}, "List Cloudformation stacks")
	router.Register("aws/cloudformation/stacks/watch", &StacksWatch{}, "Watch Stacks")
	router.Register("aws/cloudformation/stacks/resources", &StackResources{}, "Describe Stack Resources")
}
<commit_msg>fix cf watch on multiple events with same timestamp<commit_after>package cloudformation

import (
	"fmt"
	"log"
	"os"
	"sort"
	"time"

	"github.com/dynport/dgtk/cli"
	"github.com/dynport/gocli"
	"github.com/dynport/gocloud/aws/cloudformation"
)

var logger = log.New(os.Stdout, "", 0)

type StacksDelete struct {
	Name string `cli:"arg required"`
}

func (d *StacksDelete) Run() error {
	logger.Printf("deleting stack %q", d.Name)
	return client.DeleteStack(d.Name)
}

type StacksList struct {
	IncludeDeleted bool `cli:"opt --deleted"`
	Full           bool `cli:"opt --full"`
}

var client = cloudformation.NewFromEnv()

func (list *StacksList) Run() error {
	rsp, e := client.ListStacks(nil)
	if e != nil {
		return e
	}
	t := gocli.NewTable()
	for _, s := range rsp.ListStacksResult.Stacks {
		if !list.IncludeDeleted && s.StackStatus == "DELETE_COMPLETE" {
			continue
		}
		parts := []interface{}{s.StackName, s.StackStatus, s.CreationTime.Format(time.RFC3339)}
		if list.Full {
			parts = append(parts, s.StackId)
		}
		t.Add(parts...)
	}
	fmt.Println(t)
	return nil
}

type StackResources struct {
	Name string `cli:"arg required"`
}

func (r *StackResources) Run() error {
	rsp, e := client.DescribeStackResources(cloudformation.DescribeStackResourcesParameters{
		StackName: r.Name,
	})
	if e != nil {
		return e
	}
	t := gocli.NewTable()
	for _, r := range rsp.DescribeStackResourcesResult.StackResources {
		t.Add(r.LogicalResourceId, r.PhysicalResourceId)
	}
	fmt.Println(t)
	return nil
}

type StackEventsList []*cloudformation.StackEvent

func (list StackEventsList) Len() int {
	return len(list)
}

func (list StackEventsList) Swap(a, b int) {
	list[a], list[b] = list[b], list[a]
}

func (list StackEventsList) Less(a, b int) bool {
	return list[a].Timestamp.Before(list[b].Timestamp)
}

var (
	ReasonUserInitiated             = "User Initiated"
	ReasonResourceCreationInitiated = "Resource creation Initiated"
)

type StacksWatch struct {
	Name    string `cli:"arg required"`
	Refresh int    `cli:"opt --refresh default=5"`
}

func (s *StacksWatch) Run() error {
	seen := map[string]struct{}{}
	for {
		rsp, e := client.DescribeStackEvents(&cloudformation.DescribeStackEventsParameters{StackName: s.Name})
		if e != nil {
			fmt.Printf("ERROR: %s\n", e.Error())
		} else {
			events := StackEventsList(rsp.DescribeStackEventsResult.StackEvents)
			sort.Sort(events)
			for _, e := range events {
				_, ok := seen[e.EventId]
				if !ok {
					ph := e.PhysicalResourceId
					fmt.Printf("%s %-24s %-32s %s\n", e.Timestamp.Format(time.RFC3339), maxLen(e.LogicalResourceId, 24), maxLen(ph, 32), e.ResourceStatus)
					switch e.ResourceStatusReason {
					case "", ReasonUserInitiated, ReasonResourceCreationInitiated:
						//
					default:
						fmt.Printf("%20s %s\n", "", gocli.Red(e.ResourceStatusReason))
					}
					seen[e.EventId] = struct{}{}
				}
			}
		}
		time.Sleep(time.Duration(s.Refresh) * time.Second)
	}
	return nil
}

func maxLen(s string, i int) string {
	if len(s) < i {
		return s
	}
	return s[0:i]
}

func Register(router *cli.Router) {
	router.Register("aws/cloudformation/stacks/delete", &StacksDelete{}, "Delete Stack")
	router.Register("aws/cloudformation/stacks/list", &StacksList{}, "List Cloudformation stacks")
	router.Register("aws/cloudformation/stacks/watch", &StacksWatch{}, "Watch Stacks")
	router.Register("aws/cloudformation/stacks/resources", &StackResources{}, "Describe Stack Resources")
}
<|endoftext|>


<commit_before>package handler

import (
	"github.com/codegangsta/cli"
	"github.com/gorilla/mux"
	accounts "github.com/ory-am/hydra/account/handler"
	"github.com/ory-am/hydra/jwt"
	"github.com/ory-am/hydra/middleware/host"
	middleware "github.com/ory-am/hydra/middleware/host"
	clients "github.com/ory-am/hydra/oauth/client/handler"
	connections "github.com/ory-am/hydra/oauth/connection/handler"
	oauth "github.com/ory-am/hydra/oauth/handler"
	"github.com/ory-am/hydra/oauth/provider"
	policies "github.com/ory-am/hydra/policy/handler"
	"github.com/ory-am/ladon/guard"

	"fmt"
	"golang.org/x/net/http2"
	"net/http"
)

type Core struct {
	Ctx               *Context
	accountHandler    *accounts.Handler
	clientHandler     *clients.Handler
	connectionHandler *connections.Handler
	oauthHandler      *oauth.Handler
	policyHandler     *policies.Handler

	guard     guard.Guarder
	providers provider.Registry

	issuer   string
	audience string
}

func (c *Core) Start(ctx *cli.Context) error {
	c.Ctx.Start()

	private, err := jwt.LoadCertificate(jwtPrivateKeyPath)
	if err != nil {
		return fmt.Errorf("Could not load private key: %s", err)
	}

	public, err := jwt.LoadCertificate(jwtPublicKeyPath)
	if err != nil {
		return fmt.Errorf("Could not load public key: %s", err)
	}

	j := jwt.New(private, public)
	m := middleware.New(c.Ctx.Policies, j)
	c.guard = new(guard.Guard)
	c.accountHandler = accounts.NewHandler(c.Ctx.Accounts, m)
	c.clientHandler = clients.NewHandler(c.Ctx.Osins, m)
	c.connectionHandler = connections.NewHandler(c.Ctx.Connections, m)
	c.providers = provider.NewRegistry(providers)
	c.policyHandler = policies.NewHandler(c.Ctx.Policies, m, c.guard, j)
	c.oauthHandler = &oauth.Handler{
		Accounts:       c.Ctx.Accounts,
		Policies:       c.Ctx.Policies,
		Guard:          c.guard,
		Connections:    c.Ctx.Connections,
		Providers:      c.providers,
		Issuer:         c.issuer,
		Audience:       c.audience,
		JWT:            j,
		OAuthConfig:    oauth.DefaultConfig(),
		OAuthStore:     c.Ctx.Osins,
		States:         c.Ctx.States,
		SignUpLocation: locations["signUp"],
		SignInLocation: locations["signIn"],
		Middleware:     host.New(c.Ctx.Policies, j),
	}

	extractor := m.ExtractAuthentication
	router := mux.NewRouter()
	c.accountHandler.SetRoutes(router, extractor)
	c.connectionHandler.SetRoutes(router, extractor)
	c.clientHandler.SetRoutes(router, extractor)
	c.oauthHandler.SetRoutes(router, extractor)

	if forceHTTP == "force" {
		http.Handle("/", router)
		http.ListenAndServe(listenOn, nil)
		return nil
	}

	http.Handle("/", router)
	srv := &http.Server{
		Addr: listenOn,
	}
	http2.ConfigureServer(srv, &http2.Server{})
	err = srv.ListenAndServeTLS(tlsCertPath, tlsKeyPath)
	if err != nil {
		return fmt.Errorf("Could not serve HTTP/2 server because %s", err)
	}
	return nil
}
<commit_msg>cli: attached policy handler to router<commit_after>package handler

import (
	"github.com/codegangsta/cli"
	"github.com/gorilla/mux"
	accounts "github.com/ory-am/hydra/account/handler"
	"github.com/ory-am/hydra/jwt"
	"github.com/ory-am/hydra/middleware/host"
	middleware "github.com/ory-am/hydra/middleware/host"
	clients "github.com/ory-am/hydra/oauth/client/handler"
	connections "github.com/ory-am/hydra/oauth/connection/handler"
	oauth "github.com/ory-am/hydra/oauth/handler"
	"github.com/ory-am/hydra/oauth/provider"
	policies "github.com/ory-am/hydra/policy/handler"
	"github.com/ory-am/ladon/guard"

	"fmt"
	"golang.org/x/net/http2"
	"net/http"
)

type Core struct {
	Ctx               *Context
	accountHandler    *accounts.Handler
	clientHandler     *clients.Handler
	connectionHandler *connections.Handler
	oauthHandler      *oauth.Handler
	policyHandler     *policies.Handler

	guard     guard.Guarder
	providers provider.Registry

	issuer   string
	audience string
}

func (c *Core) Start(ctx *cli.Context) error {
	c.Ctx.Start()

	private, err := jwt.LoadCertificate(jwtPrivateKeyPath)
	if err != nil {
		return fmt.Errorf("Could not load private key: %s", err)
	}

	public, err := jwt.LoadCertificate(jwtPublicKeyPath)
	if err != nil {
		return fmt.Errorf("Could not load public key: %s", err)
	}

	j := jwt.New(private, public)
	m := middleware.New(c.Ctx.Policies, j)
	c.guard = new(guard.Guard)
	c.accountHandler = accounts.NewHandler(c.Ctx.Accounts, m)
	c.clientHandler = clients.NewHandler(c.Ctx.Osins, m)
	c.connectionHandler = connections.NewHandler(c.Ctx.Connections, m)
	c.providers = provider.NewRegistry(providers)
	c.policyHandler = policies.NewHandler(c.Ctx.Policies, m, c.guard, j)
	c.oauthHandler = &oauth.Handler{
		Accounts:       c.Ctx.Accounts,
		Policies:       c.Ctx.Policies,
		Guard:          c.guard,
		Connections:    c.Ctx.Connections,
		Providers:      c.providers,
		Issuer:         c.issuer,
		Audience:       c.audience,
		JWT:            j,
		OAuthConfig:    oauth.DefaultConfig(),
		OAuthStore:     c.Ctx.Osins,
		States:         c.Ctx.States,
		SignUpLocation: locations["signUp"],
		SignInLocation: locations["signIn"],
		Middleware:     host.New(c.Ctx.Policies, j),
	}

	extractor := m.ExtractAuthentication
	router := mux.NewRouter()
	c.accountHandler.SetRoutes(router, extractor)
	c.connectionHandler.SetRoutes(router, extractor)
	c.clientHandler.SetRoutes(router, extractor)
	c.oauthHandler.SetRoutes(router, extractor)
	c.policyHandler.SetRoutes(router, extractor)

	if forceHTTP == "force" {
		http.Handle("/", router)
		http.ListenAndServe(listenOn, nil)
		return nil
	}

	http.Handle("/", router)
	srv := &http.Server{
		Addr: listenOn,
	}
	http2.ConfigureServer(srv, &http2.Server{})
	err = srv.ListenAndServeTLS(tlsCertPath, tlsKeyPath)
	if err != nil {
		return fmt.Errorf("Could not serve HTTP/2 server because %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2010-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"github.com/hoisie/web"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func dirList(dir string) string {
	out := `

		<title>` + dir + `</title>

` + dir + `

`
	list, err := ioutil.ReadDir(dir)
	if err == nil {
		for _, v := range list {
			name := v.Name()
			if v.IsDir() {
				out += "
" + name + "/
"
			} else if strings.HasSuffix(name, ".md") {
				out += "
" + name + "
"
			}
		}
	} else {
		log.Println("error:", err)
		return "404: Not Found"
	}
	out += `

`
	return out
}

func mkServer(contextDir string, format bool) func(*web.Context, string) string {
	return func(params *web.Context, val string) string {
		if len(val) == 0 || (len(val) == 1 && val == "/") {
			val = "."
		}
		fullPath := contextDir + val
		if fi, err := os.Stat(fullPath); err != nil {
			log.Println("error:", err)
			return "404: File not found: " + val
		} else if fi.IsDir() {
			return dirList(fullPath)
		} else {
			file, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Println("error:", err)
				return "404: File not found: " + val
			} else if strings.HasSuffix(val, ".md") {
				var text string
				if format {
					text = string(blackfriday.MarkdownBasic(file))
				} else {
					text = string(file)
				}
				return `

		<title>` + val + `</title>

` + text + `

`
			}
			return string(file)
		}
	}
}

func main() {
	var contextDir = ""
	global := flag.Bool("global", false, "Allow the server to access any files that the user running it has access to.")
	remote := flag.Bool("remote", false, "Allow the remote clients to access the server.")
	port := "15448"
	flag.Parse()
	//read the context from the input and override whats in the settings file if something was there
	if flag.NArg() != 0 {
		contextDir = flag.Arg(0)
	}
	//make sure the context is a directory
	if len(contextDir) != 0 && !strings.HasSuffix(contextDir, "/") {
		contextDir += "/"
	}

	contextServe := mkServer(contextDir, true)
	rawServe := mkServer(contextDir, false)
	globalServe := mkServer("", true)
	web.Get("/doc/(.*)", contextServe)
	web.Get("/doc", contextServe)
	web.Get("/raw/(.*)", rawServe)
	web.Get("/raw", rawServe)
	if *global {
		web.Get("/doc-g/(.*)", globalServe)
		web.Get("/doc-g", globalServe)
	}
	if *remote {
		web.Run("0.0.0.0:" + port)
	} else {
		web.Run("127.0.0.1:" + port)
	}
}
<commit_msg>Fixed issue with directory navigator not opening files correctly.<commit_after>/*
   Copyright 2010-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"github.com/hoisie/web"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func dirList(dir string) string {
	out := `

		<title>` + dir + `</title>

` + dir + `

`
	list, err := ioutil.ReadDir(dir)
	if err == nil {
		for _, v := range list {
			name := v.Name()
			if v.IsDir() {
				out += "
" + name + "/
"
			} else if strings.HasSuffix(name, ".md") {
				out += "
" + name + "
"
			}
		}
	} else {
		log.Println("error:", err)
		return "404: Not Found"
	}
	out += `

`
	return out
}

func mkServer(contextDir string, format bool) func(*web.Context, string) string {
	return func(params *web.Context, val string) string {
		if len(val) == 0 || (len(val) == 1 && val == "/") {
			val = "."
		}
		fullPath := contextDir + val
		if fi, err := os.Stat(fullPath); err != nil {
			log.Println("error:", err)
			return "404: File not found: " + val
		} else if fi.IsDir() {
			return dirList(fullPath)
		} else {
			file, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Println("error:", err)
				return "404: File not found: " + val
			} else if strings.HasSuffix(val, ".md") {
				var text string
				if format {
					text = string(blackfriday.MarkdownBasic(file))
				} else {
					text = string(file)
				}
				return `

		<title>` + val + `</title>

` + text + `

`
			}
			return string(file)
		}
	}
}

func main() {
	var contextDir = ""
	global := flag.Bool("global", false, "Allow the server to access any files that the user running it has access to.")
	remote := flag.Bool("remote", false, "Allow the remote clients to access the server.")
	port := "15448"
	flag.Parse()
	//read the context from the input and override whats in the settings file if something was there
	if flag.NArg() != 0 {
		contextDir = flag.Arg(0)
	}
	//make sure the context is a directory
	if len(contextDir) != 0 && !strings.HasSuffix(contextDir, "/") {
		contextDir += "/"
	}

	contextServe := mkServer(contextDir, true)
	rawServe := mkServer(contextDir, false)
	globalServe := mkServer("", true)
	web.Get("/doc/(.*)", contextServe)
	web.Get("/doc", contextServe)
	web.Get("/raw/(.*)", rawServe)
	web.Get("/raw", rawServe)
	if *global {
		web.Get("/doc-g/(.*)", globalServe)
		web.Get("/doc-g", globalServe)
	}
	if *remote {
		web.Run("0.0.0.0:" + port)
	} else {
		web.Run("127.0.0.1:" + port)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2010-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"github.com/hoisie/web"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func dirList() string {
	out := `

		<title>Directory Listing</title>

Docs

`
	list, err := ioutil.ReadDir(".")
	if err == nil {
		for _, v := range list {
			name := v.Name()
			if name[len(name)-3:] == ".md" {
				out += "
" + name + "
"
			}
		}
	} else {
		log.Println("error:", err)
		return "404: Not Found"
	}
	out += `

`
	return out
}

func mkServer(contextDir string, format bool) func(*web.Context, string) string {
	return func(params *web.Context, val string) string {
		if len(val) == 0 || (len(val) == 1 && val == "/") {
			val = "."
		}
		if fi, err := os.Stat(contextDir + val); fi.IsDir() {
			return dirList()
		} else if err != nil {
			log.Println("error:", err)
			return "404: File not found: " + val
		} else {
			file, err := ioutil.ReadFile(contextDir + val)
			if err != nil {
				log.Println("error:", err)
				return "404: File not found: " + val
			} else if strings.HasSuffix(val, ".md") {
				var text string
				if format {
					text = string(blackfriday.MarkdownBasic(file))
				} else {
					text = string(file)
				}
				return `

		<title>` + val + `</title>

` + text + `

`
			}
			return string(file)
		}
	}
}

func main() {
	var contextDir = ""
	global := flag.Bool("global", false, "Allow the server to access any files that the user running it has access to.")
	remote := flag.Bool("remote", false, "Allow the remote clients to access the server.")
	port := "15448"
	flag.Parse()
	//read the context from the input and override whats in the settings file if something was there
	if flag.NArg() != 0 {
		contextDir = flag.Arg(0)
	}
	//make sure the context is a directory
	if len(contextDir) != 0 && !strings.HasSuffix(contextDir, "/") {
		contextDir += "/"
	}

	contextServe := mkServer(contextDir, true)
	rawServe := mkServer(contextDir, false)
	globalServe := mkServer("", true)
	web.Get("/doc/(.*)", contextServe)
	web.Get("/doc", contextServe)
	web.Get("/raw/(.*)", rawServe)
	web.Get("/raw", rawServe)
	if *global {
		web.Get("/doc-g/(.*)", globalServe)
		web.Get("/doc-g", globalServe)
	}
	if *remote {
		web.Run("0.0.0.0:" + port)
	} else {
		web.Run("127.0.0.1:" + port)
	}
}
<commit_msg>Made directory listing recursive.<commit_after>/*
   Copyright 2010-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"github.com/hoisie/web"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func dirList(dir string) string {
	out := `

		<title>` + dir + `</title>

` + dir + `

`
	list, err := ioutil.ReadDir(dir)
	if err == nil {
		for _, v := range list {
			name := v.Name()
			if v.IsDir() || name[len(name)-3:] == ".md" {
				out += "
" + name + "
"
			}
		}
	} else {
		log.Println("error:", err)
		return "404: Not Found"
	}
	out += `

`
	return out
}

func mkServer(contextDir string, format bool) func(*web.Context, string) string {
	return func(params *web.Context, val string) string {
		if len(val) == 0 || (len(val) == 1 && val == "/") {
			val = "."
		}
		fullPath := contextDir + val
		if fi, err := os.Stat(fullPath); err == nil && fi.IsDir() {
			return dirList(fullPath)
		} else if err != nil {
			log.Println("error:", err)
			return "404: File not found: " + val
		} else {
			file, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Println("error:", err)
				return "404: File not found: " + val
			} else if strings.HasSuffix(val, ".md") {
				var text string
				if format {
					text = string(blackfriday.MarkdownBasic(file))
				} else {
					text = string(file)
				}
				return `

		<title>` + val + `</title>

` + text + `

`
			}
			return string(file)
		}
	}
}

func main() {
	var contextDir = ""
	global := flag.Bool("global", false, "Allow the server to access any files that the user running it has access to.")
	remote := flag.Bool("remote", false, "Allow the remote clients to access the server.")
	port := "15448"
	flag.Parse()
	//read the context from the input and override whats in the settings file if something was there
	if flag.NArg() != 0 {
		contextDir = flag.Arg(0)
	}
	//make sure the context is a directory
	if len(contextDir) != 0 && !strings.HasSuffix(contextDir, "/") {
		contextDir += "/"
	}

	contextServe := mkServer(contextDir, true)
	rawServe := mkServer(contextDir, false)
	globalServe := mkServer("", true)
	web.Get("/doc/(.*)", contextServe)
	web.Get("/doc", contextServe)
	web.Get("/raw/(.*)", rawServe)
	web.Get("/raw", rawServe)
	if *global {
		web.Get("/doc-g/(.*)", globalServe)
		web.Get("/doc-g", globalServe)
	}
	if *remote {
		web.Run("0.0.0.0:" + port)
	} else {
		web.Run("127.0.0.1:" + port)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bufio"
	"regexp"

	"io"

	"strings"

	"github.com/FreeFeed/clio-restore/internal/clio"
	"github.com/davidmz/mustbe"
)

type localFile struct {
	*zip.File
	OrigName string
}

func (a *App) restoreThumbnails(entry *clio.Entry) (resUIDs []string) {
	if len(entry.Thumbnails) == 0 {
		return
	}

	bodyLinks := make(map[string]bool)
	for _, l := range entry.Links {
		bodyLinks[l] = true
	}

	// All images is of known types
	{
		handlableOnly := true
		for _, t := range entry.Thumbnails {
			if !(ffMediaURLRe.MatchString(t.Link) ||
				strings.HasPrefix(t.Link, "http://friendfeed.com/e/") ||
				strings.HasPrefix(t.URL, "http://twitpic.com/show/thumb/") ||
				imgurRe.MatchString(t.URL)) {
				handlableOnly = false
				break
			}
		}
		if handlableOnly {
			for _, t := range entry.Thumbnails {
				if ffMediaURLRe.MatchString(t.Link) {
					// get local file
					if uid, ok := a.createImageAttachment(t.Link); ok {
						resUIDs = append(resUIDs, uid)
					}
				}
				if t.Player != nil {
					// do nothing
				}
				if strings.HasPrefix(t.URL, "http://twitpic.com/show/thumb/") {
					url := strings.Replace(t.URL, "/thumb/", "/large/", 1)
					if uid, ok := a.createImageAttachment(url); ok {
						resUIDs = append(resUIDs, uid)
					}
				}
				if imgurRe.MatchString(t.URL) {
					code := imgurRe.FindStringSubmatch(t.URL)[1]
					if uid, ok := a.createImageAttachment("http://i.imgur.com/" + code + ".jpg"); ok {
						resUIDs = append(resUIDs, uid)
					}
				}
			}
			return
		}
	}

	// Dead services
	if strings.HasPrefix(entry.Via.URL, "http://filmfeed.ru/users/") ||
		strings.HasPrefix(entry.Via.URL, "http://www.zooomr.com/") ||
		strings.HasPrefix(entry.Via.URL, "http://meme.yahoo.com/") ||
		false {
		return
	}

	// Bookmarklet or direct post
	if entry.Via.URL == "http://friendfeed.com/share/bookmarklet" || entry.Via.URL == clio.DefaultViaURL {
		isSameURL := true
		isLocalThumbs := true
		for _, t := range entry.Thumbnails {
			if t.Link != entry.Thumbnails[0].Link {
				isSameURL = false
				break
			}
			if !ffMediaURLRe.MatchString(t.URL) {
				isLocalThumbs = false
				break
			}
		}

		if isSameURL && isLocalThumbs {
			// All links is the same
			// Using local thumbnails
			for _, t := range entry.Thumbnails {
				if uid, ok := a.createImageAttachment(t.URL); ok {
					resUIDs = append(resUIDs, uid)
				}
			}
			if !bodyLinks[entry.Thumbnails[0].Link] {
				// Add link if body doesn't contan it
				entry.Body += " - " + entry.Thumbnails[0].Link
			}
			return
		}
	}

	// Flickr
	{
		isAllFlickr := true
		isInBody := false
		for _, t := range entry.Thumbnails {
			if !strings.HasPrefix(t.Link, "http://www.flickr.com/photos/") {
				isAllFlickr = false
				break
			}
			if bodyLinks[t.Link] {
				isInBody = true
			}
		}
		if isAllFlickr && isInBody {
			// do nothing
			return
		}
	}

	// fotki.yandex.ru
	if strings.HasPrefix(entry.Via.URL, "http://fotki.yandex.ru/users/") {
		for _, t := range entry.Thumbnails {
			if strings.HasPrefix(t.URL, "http://img-fotki.yandex.ru/get/") && strings.HasPrefix(t.Link, "http://fotki.yandex.ru/users/") {
				imgURL := t.URL[:len(t.URL)-1] + "orig"
				if uid, ok := a.createImageAttachment(imgURL); ok {
					resUIDs = append(resUIDs, uid)
				}
			}
		}
		return
	}

	// http://picasaweb.google.com/
	if strings.HasPrefix(entry.Via.URL, "http://picasaweb.google.com/") {
		// импортируем картинку, которая присутствует в теле поста, в полном размере
		for _, t := range entry.Thumbnails {
			if picasaImageRe.MatchString(t.URL) && bodyLinks[t.Link] {
				url := strings.Replace(t.URL, "/s144/", "/", 1)
				if uid, ok := a.createImageAttachment(url); ok {
					resUIDs = append(resUIDs, uid)
				}
			}
		}
		return
	}

	// If there is only one thumb
	if len(entry.Thumbnails) == 1 {
		th := entry.Thumbnails[0]

		if strings.HasPrefix(th.Link, "http://www.youtube.com/watch") {
			// do nothing
			return
		}

		if strings.HasPrefix(th.Link, "http://vimeo.com/") && bodyLinks[th.Link] {
			// do nothing
			return
		}

		if instagramImageRe.MatchString(th.Link) {
			for _, l := range entry.Links {
				if strings.HasPrefix(l, "http://instagr.am/p/") {
					// do nothing
					return
				}
			}
		}

		if strings.HasPrefix(th.Link, "http://behance.vo.llnwd.net/") {
			for _, l := range entry.Links {
				if strings.HasPrefix(l, "http://www.behance.net/gallery/") {
					// do nothing
					return
				}
			}
		}

		if strings.HasPrefix(th.Link, "http://b.vimeocdn.com/ts/") {
			for _, l := range entry.Links {
				if strings.HasPrefix(l, "http://vimeo.com/") || strings.HasPrefix(l, "https://vimeo.com/") {
					// do nothing
					return
				}
			}
		}
	}

	// Common case
	for _, t := range entry.Thumbnails {
		if ffMediaURLRe.MatchString(t.Link) {
			// get local file
			if uid, ok := a.createImageAttachment(t.Link); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if t.Player != nil {
			// do nothing
		} else if strings.HasPrefix(t.URL, "http://twitpic.com/show/thumb/") {
			url := strings.Replace(t.URL, "/thumb/", "/large/", 1)
			if uid, ok := a.createImageAttachment(url); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if imgurRe.MatchString(t.URL) {
			code := imgurRe.FindStringSubmatch(t.URL)[1]
			if uid, ok := a.createImageAttachment("http://i.imgur.com/" + code + ".jpg"); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if soupImageRe.MatchString(t.URL) {
			if uid, ok := a.createImageAttachment(strings.Replace(t.URL, "_400.gif", ".gif", 1)); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else {
			if uid, ok := a.createImageAttachment(t.Link, t.URL); ok {
				resUIDs = append(resUIDs, uid)
			}
		}
	}

	return
}

func (a *App) readImageFiles() {
	a.ImageFiles = make(map[string]*localFile)
	name2id := make(map[string]string) // file name -> file UID

	var (
		tsvFileRe   = regexp.MustCompile(`^[a-z0-9-]+/_json/data/images\.tsv$`)
		mediaURLRe  = regexp.MustCompile(`[0-9a-f]+$`)
		imageFileRe = regexp.MustCompile(`^[a-z0-9-]+/images/media/([^/]+)$`)
		thumbFileRe = regexp.MustCompile(`^[a-z0-9-]+/images/media/thumbnails/(([0-9a-f]+).+)`)
	)

	// Looking for the TSV file
	for _, f := range a.ZipFiles {
		if tsvFileRe.MatchString(f.Name) {
			r := mustbe.OKVal(f.Open()).(io.ReadCloser)
			scanner := bufio.NewScanner(r)
			for scanner.Scan() {
				parts := strings.SplitN(scanner.Text(), "\t", 2)
				if len(parts) != 2 {
					continue
				}
				m := mediaURLRe.FindStringSubmatch(parts[0])
				if m == nil {
					continue
				}
				name2id[parts[1]] = m[0]
			}
			r.Close()
		}
	}

	// Now looking for images
	for _, f := range a.ZipFiles {
		if imageFileRe.MatchString(f.Name) {
			name := imageFileRe.FindStringSubmatch(f.Name)[1]
			if id, ok := name2id[name]; ok {
				a.ImageFiles[id] = &localFile{File: f, OrigName: name}
			}
		}
		if thumbFileRe.MatchString(f.Name) {
			m := thumbFileRe.FindStringSubmatch(f.Name)
			a.ImageFiles[m[2]] = &localFile{File: f, OrigName: m[1]}
		}
	}

	return
}
<commit_msg>Load large version of  pbs.twimg.com images<commit_after>package main

import (
	"archive/zip"
	"bufio"
	"regexp"

	"io"

	"strings"

	"github.com/FreeFeed/clio-restore/internal/clio"
	"github.com/davidmz/mustbe"
)

type localFile struct {
	*zip.File
	OrigName string
}

func (a *App) restoreThumbnails(entry *clio.Entry) (resUIDs []string) {
	if len(entry.Thumbnails) == 0 {
		return
	}

	bodyLinks := make(map[string]bool)
	for _, l := range entry.Links {
		bodyLinks[l] = true
	}

	// All images is of known types
	{
		handlableOnly := true
		for _, t := range entry.Thumbnails {
			if !(ffMediaURLRe.MatchString(t.Link) ||
				strings.HasPrefix(t.Link, "http://friendfeed.com/e/") ||
				strings.HasPrefix(t.URL, "http://twitpic.com/show/thumb/") ||
				imgurRe.MatchString(t.URL)) {
				handlableOnly = false
				break
			}
		}
		if handlableOnly {
			for _, t := range entry.Thumbnails {
				if ffMediaURLRe.MatchString(t.Link) {
					// get local file
					if uid, ok := a.createImageAttachment(t.Link); ok {
						resUIDs = append(resUIDs, uid)
					}
				}
				if t.Player != nil {
					// do nothing
				}
				if strings.HasPrefix(t.URL, "http://twitpic.com/show/thumb/") {
					url := strings.Replace(t.URL, "/thumb/", "/large/", 1)
					if uid, ok := a.createImageAttachment(url); ok {
						resUIDs = append(resUIDs, uid)
					}
				}
				if imgurRe.MatchString(t.URL) {
					code := imgurRe.FindStringSubmatch(t.URL)[1]
					if uid, ok := a.createImageAttachment("http://i.imgur.com/" + code + ".jpg"); ok {
						resUIDs = append(resUIDs, uid)
					}
				}
			}
			return
		}
	}

	// Dead services
	if strings.HasPrefix(entry.Via.URL, "http://filmfeed.ru/users/") ||
		strings.HasPrefix(entry.Via.URL, "http://www.zooomr.com/") ||
		strings.HasPrefix(entry.Via.URL, "http://meme.yahoo.com/") ||
		false {
		return
	}

	// Bookmarklet or direct post
	if entry.Via.URL == "http://friendfeed.com/share/bookmarklet" || entry.Via.URL == clio.DefaultViaURL {
		isSameURL := true
		isLocalThumbs := true
		for _, t := range entry.Thumbnails {
			if t.Link != entry.Thumbnails[0].Link {
				isSameURL = false
				break
			}
			if !ffMediaURLRe.MatchString(t.URL) {
				isLocalThumbs = false
				break
			}
		}

		if isSameURL && isLocalThumbs {
			// All links is the same
			// Using local thumbnails
			for _, t := range entry.Thumbnails {
				if uid, ok := a.createImageAttachment(t.URL); ok {
					resUIDs = append(resUIDs, uid)
				}
			}
			if !bodyLinks[entry.Thumbnails[0].Link] {
				// Add link if body doesn't contan it
				entry.Body += " - " + entry.Thumbnails[0].Link
			}
			return
		}
	}

	// Flickr
	{
		isAllFlickr := true
		isInBody := false
		for _, t := range entry.Thumbnails {
			if !strings.HasPrefix(t.Link, "http://www.flickr.com/photos/") {
				isAllFlickr = false
				break
			}
			if bodyLinks[t.Link] {
				isInBody = true
			}
		}
		if isAllFlickr && isInBody {
			// do nothing
			return
		}
	}

	// fotki.yandex.ru
	if strings.HasPrefix(entry.Via.URL, "http://fotki.yandex.ru/users/") {
		for _, t := range entry.Thumbnails {
			if strings.HasPrefix(t.URL, "http://img-fotki.yandex.ru/get/") && strings.HasPrefix(t.Link, "http://fotki.yandex.ru/users/") {
				imgURL := t.URL[:len(t.URL)-1] + "orig"
				if uid, ok := a.createImageAttachment(imgURL); ok {
					resUIDs = append(resUIDs, uid)
				}
			}
		}
		return
	}

	// http://picasaweb.google.com/
	if strings.HasPrefix(entry.Via.URL, "http://picasaweb.google.com/") {
		// импортируем картинку, которая присутствует в теле поста, в полном размере
		for _, t := range entry.Thumbnails {
			if picasaImageRe.MatchString(t.URL) && bodyLinks[t.Link] {
				url := strings.Replace(t.URL, "/s144/", "/", 1)
				if uid, ok := a.createImageAttachment(url); ok {
					resUIDs = append(resUIDs, uid)
				}
			}
		}
		return
	}

	// If there is only one thumb
	if len(entry.Thumbnails) == 1 {
		th := entry.Thumbnails[0]

		if strings.HasPrefix(th.Link, "http://www.youtube.com/watch") {
			// do nothing
			return
		}

		if strings.HasPrefix(th.Link, "http://vimeo.com/") && bodyLinks[th.Link] {
			// do nothing
			return
		}

		if instagramImageRe.MatchString(th.Link) {
			for _, l := range entry.Links {
				if strings.HasPrefix(l, "http://instagr.am/p/") {
					// do nothing
					return
				}
			}
		}

		if strings.HasPrefix(th.Link, "http://behance.vo.llnwd.net/") {
			for _, l := range entry.Links {
				if strings.HasPrefix(l, "http://www.behance.net/gallery/") {
					// do nothing
					return
				}
			}
		}

		if strings.HasPrefix(th.Link, "http://b.vimeocdn.com/ts/") {
			for _, l := range entry.Links {
				if strings.HasPrefix(l, "http://vimeo.com/") || strings.HasPrefix(l, "https://vimeo.com/") {
					// do nothing
					return
				}
			}
		}
	}

	// Common case
	for _, t := range entry.Thumbnails {
		if ffMediaURLRe.MatchString(t.Link) {
			// get local file
			if uid, ok := a.createImageAttachment(t.Link); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if t.Player != nil {
			// do nothing
		} else if strings.HasPrefix(t.URL, "http://twitpic.com/show/thumb/") {
			url := strings.Replace(t.URL, "/thumb/", "/large/", 1)
			if uid, ok := a.createImageAttachment(url); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if strings.HasPrefix(t.Link, "http://pbs.twimg.com/media/") {
			if uid, ok := a.createImageAttachment(t.Link+":large", t.URL); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if imgurRe.MatchString(t.URL) {
			code := imgurRe.FindStringSubmatch(t.URL)[1]
			if uid, ok := a.createImageAttachment("http://i.imgur.com/" + code + ".jpg"); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else if soupImageRe.MatchString(t.URL) {
			if uid, ok := a.createImageAttachment(strings.Replace(t.URL, "_400.gif", ".gif", 1)); ok {
				resUIDs = append(resUIDs, uid)
			}
		} else {
			if uid, ok := a.createImageAttachment(t.Link, t.URL); ok {
				resUIDs = append(resUIDs, uid)
			}
		}
	}

	return
}

func (a *App) readImageFiles() {
	a.ImageFiles = make(map[string]*localFile)
	name2id := make(map[string]string) // file name -> file UID

	var (
		tsvFileRe   = regexp.MustCompile(`^[a-z0-9-]+/_json/data/images\.tsv$`)
		mediaURLRe  = regexp.MustCompile(`[0-9a-f]+$`)
		imageFileRe = regexp.MustCompile(`^[a-z0-9-]+/images/media/([^/]+)$`)
		thumbFileRe = regexp.MustCompile(`^[a-z0-9-]+/images/media/thumbnails/(([0-9a-f]+).+)`)
	)

	// Looking for the TSV file
	for _, f := range a.ZipFiles {
		if tsvFileRe.MatchString(f.Name) {
			r := mustbe.OKVal(f.Open()).(io.ReadCloser)
			scanner := bufio.NewScanner(r)
			for scanner.Scan() {
				parts := strings.SplitN(scanner.Text(), "\t", 2)
				if len(parts) != 2 {
					continue
				}
				m := mediaURLRe.FindStringSubmatch(parts[0])
				if m == nil {
					continue
				}
				name2id[parts[1]] = m[0]
			}
			r.Close()
		}
	}

	// Now looking for images
	for _, f := range a.ZipFiles {
		if imageFileRe.MatchString(f.Name) {
			name := imageFileRe.FindStringSubmatch(f.Name)[1]
			if id, ok := name2id[name]; ok {
				a.ImageFiles[id] = &localFile{File: f, OrigName: name}
			}
		}
		if thumbFileRe.MatchString(f.Name) {
			m := thumbFileRe.FindStringSubmatch(f.Name)
			a.ImageFiles[m[2]] = &localFile{File: f, OrigName: m[1]}
		}
	}

	return
}
<|endoftext|>


<commit_before>// Package mountinfo provides a set of functions to retrieve information about OS mounts.
//
// Currently it supports Linux. For historical reasons, there is also some support for FreeBSD,
// and a shallow implementation for Windows, but in general this is Linux-only package, so
// the rest of the document only applies to Linux, unless explicitly specified otherwise.
//
// In Linux, information about mounts seen by the current process is available from
// /proc/self/mountinfo. Note that due to mount namespaces, different processes can
// see different mounts. A per-process mountinfo table is available from /proc/<PID>/mountinfo,
// where <PID> is a numerical process identifier.
//
// In general, /proc is not a very efficient interface, and mountinfo is not an exception.
// For example, there is no way to get information about a specific mount point (i.e. it
// is all-or-nothing). This package tries to hide the /proc ineffectiveness by using
// parse filters while reading mountinfo. A filter can skip some entries, or stop
// processing the rest of the file once the needed information is found.
//
// For mountinfo filters that accept path as an argument, the path must be:
//  - absolute;
//  - having all symlinks resolved;
//  - being cleaned.
//
// One way to achieve all of the above is to employ filepath.Abs followed by
// filepath.EvalSymlinks (the latter calls filepath.Clean on the result so
// there is no need to explicitly call filepath.Clean).
//
// NOTE that in many cases there is no need to consult mountinfo at all. Here are some
// of the cases where mountinfo should not be parsed:
//
// 1. Before performing a mount. Usually, this is not needed, but if required (say to
//    prevent over-mounts), to check whether a directory is mounted, call os.Lstat
//    on it and its parent directory, and compare their st.Sys().(*syscall.Stat_t).Dev
//    fields -- if they differ, then the directory is the mount point. NOTE this does
//    not work for bind mounts. Optionally, the filesystem type can also be checked
//    by calling unix.Statfs and checking the Type field (i.e. filesystem type).
//
// 2. After performing a mount. If there is no error returned, the mount succeeded;
//    checking the mount table for a new mount is redundant and expensive.
//
// 3. Before performing an unmount. It is more efficient to do an unmount and ignore
//    a specific error (EINVAL) which tells the directory is not mounted.
//
// 4. After performing an unmount. If there is no error returned, the unmount succeeded.
//
// 5. To find the mount point root of a specific directory. You can perform os.Stat()
//    on the directory and traverse up until the Dev field of a parent directory differs.

package mountinfo
<commit_msg>mountinfo: fix go doc<commit_after>// Package mountinfo provides a set of functions to retrieve information about OS mounts.
//
// Currently it supports Linux. For historical reasons, there is also some support for FreeBSD,
// and a shallow implementation for Windows, but in general this is Linux-only package, so
// the rest of the document only applies to Linux, unless explicitly specified otherwise.
//
// In Linux, information about mounts seen by the current process is available from
// /proc/self/mountinfo. Note that due to mount namespaces, different processes can
// see different mounts. A per-process mountinfo table is available from /proc/<PID>/mountinfo,
// where <PID> is a numerical process identifier.
//
// In general, /proc is not a very efficient interface, and mountinfo is not an exception.
// For example, there is no way to get information about a specific mount point (i.e. it
// is all-or-nothing). This package tries to hide the /proc ineffectiveness by using
// parse filters while reading mountinfo. A filter can skip some entries, or stop
// processing the rest of the file once the needed information is found.
//
// For mountinfo filters that accept path as an argument, the path must be:
//  - absolute;
//  - having all symlinks resolved;
//  - being cleaned.
//
// One way to achieve all of the above is to employ filepath.Abs followed by
// filepath.EvalSymlinks (the latter calls filepath.Clean on the result so
// there is no need to explicitly call filepath.Clean).
//
// NOTE that in many cases there is no need to consult mountinfo at all. Here are some
// of the cases where mountinfo should not be parsed:
//
// 1. Before performing a mount. Usually, this is not needed, but if required (say to
//    prevent over-mounts), to check whether a directory is mounted, call os.Lstat
//    on it and its parent directory, and compare their st.Sys().(*syscall.Stat_t).Dev
//    fields -- if they differ, then the directory is the mount point. NOTE this does
//    not work for bind mounts. Optionally, the filesystem type can also be checked
//    by calling unix.Statfs and checking the Type field (i.e. filesystem type).
//
// 2. After performing a mount. If there is no error returned, the mount succeeded;
//    checking the mount table for a new mount is redundant and expensive.
//
// 3. Before performing an unmount. It is more efficient to do an unmount and ignore
//    a specific error (EINVAL) which tells the directory is not mounted.
//
// 4. After performing an unmount. If there is no error returned, the unmount succeeded.
//
// 5. To find the mount point root of a specific directory. You can perform os.Stat()
//    on the directory and traverse up until the Dev field of a parent directory differs.
package mountinfo
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/klog"
)

// ExpirationCache implements the store interface
//	1. All entries are automatically time stamped on insert
//		a. The key is computed based off the original item/keyFunc
//		b. The value inserted under that key is the timestamped item
//	2. Expiration happens lazily on read based on the expiration policy
//      a. No item can be inserted into the store while we're expiring
//		   *any* item in the cache.
//	3. Time-stamps are stripped off unexpired entries before return
// Note that the ExpirationCache is inherently slower than a normal
// threadSafeStore because it takes a write lock every time it checks if
// an item has expired.
type ExpirationCache struct {
	cacheStorage     ThreadSafeStore
	keyFunc          KeyFunc
	clock            clock.Clock
	expirationPolicy ExpirationPolicy
	// expirationLock is a write lock used to guarantee that we don't clobber
	// newly inserted objects because of a stale expiration timestamp comparison
	expirationLock sync.Mutex
}

// ExpirationPolicy dictates when an object expires. Currently only abstracted out
// so unittests don't rely on the system clock.
type ExpirationPolicy interface {
	IsExpired(obj *timestampedEntry) bool
}

// TTLPolicy implements a ttl based ExpirationPolicy.
type TTLPolicy struct {
	//	 >0: Expire entries with an age > ttl
	//	<=0: Don't expire any entry
	Ttl time.Duration

	// Clock used to calculate ttl expiration
	Clock clock.Clock
}

// IsExpired returns true if the given object is older than the ttl, or it can't
// determine its age.
func (p *TTLPolicy) IsExpired(obj *timestampedEntry) bool {
	return p.Ttl > 0 && p.Clock.Since(obj.timestamp) > p.Ttl
}

// timestampedEntry is the only type allowed in a ExpirationCache.
type timestampedEntry struct {
	obj       interface{}
	timestamp time.Time
}

// getTimestampedEntry returns the timestampedEntry stored under the given key.
func (c *ExpirationCache) getTimestampedEntry(key string) (*timestampedEntry, bool) {
	item, _ := c.cacheStorage.Get(key)
	if tsEntry, ok := item.(*timestampedEntry); ok {
		return tsEntry, true
	}
	return nil, false
}

// getOrExpire retrieves the object from the timestampedEntry if and only if it hasn't
// already expired. It holds a write lock across deletion.
func (c *ExpirationCache) getOrExpire(key string) (interface{}, bool) {
	// Prevent all inserts from the time we deem an item as "expired" to when we
	// delete it, so an un-expired item doesn't sneak in under the same key, just
	// before the Delete.
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	timestampedItem, exists := c.getTimestampedEntry(key)
	if !exists {
		return nil, false
	}
	if c.expirationPolicy.IsExpired(timestampedItem) {
		klog.V(4).Infof("Entry %v: %+v has expired", key, timestampedItem.obj)
		c.cacheStorage.Delete(key)
		return nil, false
	}
	return timestampedItem.obj, true
}

// GetByKey returns the item stored under the key, or sets exists=false.
func (c *ExpirationCache) GetByKey(key string) (interface{}, bool, error) {
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// Get returns unexpired items. It purges the cache of expired items in the
// process.
func (c *ExpirationCache) Get(obj interface{}) (interface{}, bool, error) {
	key, err := c.keyFunc(obj)
	if err != nil {
		return nil, false, KeyError{obj, err}
	}
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// List retrieves a list of unexpired items. It purges the cache of expired
// items in the process.
func (c *ExpirationCache) List() []interface{} {
	items := c.cacheStorage.List()

	list := make([]interface{}, 0, len(items))
	for _, item := range items {
		obj := item.(*timestampedEntry).obj
		if key, err := c.keyFunc(obj); err != nil {
			list = append(list, obj)
		} else if obj, exists := c.getOrExpire(key); exists {
			list = append(list, obj)
		}
	}
	return list
}

// ListKeys returns a list of all keys in the expiration cache.
func (c *ExpirationCache) ListKeys() []string {
	return c.cacheStorage.ListKeys()
}

// Add timestamps an item and inserts it into the cache, overwriting entries
// that might exist under the same key.
func (c *ExpirationCache) Add(obj interface{}) error {
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()

	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.cacheStorage.Add(key, &timestampedEntry{obj, c.clock.Now()})
	return nil
}

// Update has not been implemented yet for lack of a use case, so this method
// simply calls `Add`. This effectively refreshes the timestamp.
func (c *ExpirationCache) Update(obj interface{}) error {
	return c.Add(obj)
}

// Delete removes an item from the cache.
func (c *ExpirationCache) Delete(obj interface{}) error {
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.cacheStorage.Delete(key)
	return nil
}

// Replace will convert all items in the given list to TimestampedEntries
// before attempting the replace operation. The replace operation will
// delete the contents of the ExpirationCache `c`.
func (c *ExpirationCache) Replace(list []interface{}, resourceVersion string) error {
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	items := make(map[string]interface{}, len(list))
	ts := c.clock.Now()
	for _, item := range list {
		key, err := c.keyFunc(item)
		if err != nil {
			return KeyError{item, err}
		}
		items[key] = &timestampedEntry{item, ts}
	}
	c.cacheStorage.Replace(items, resourceVersion)
	return nil
}

// Resync will touch all objects to put them into the processing queue
func (c *ExpirationCache) Resync() error {
	return c.cacheStorage.Resync()
}

// NewTTLStore creates and returns a ExpirationCache with a TTLPolicy
func NewTTLStore(keyFunc KeyFunc, ttl time.Duration) Store {
	return &ExpirationCache{
		cacheStorage:     NewThreadSafeStore(Indexers{}, Indices{}),
		keyFunc:          keyFunc,
		clock:            clock.RealClock{},
		expirationPolicy: &TTLPolicy{ttl, clock.RealClock{}},
	}
}
<commit_msg>Reduce the duration where expirationLock is held for expiration_cache<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/klog"
)

// ExpirationCache implements the store interface
//	1. All entries are automatically time stamped on insert
//		a. The key is computed based off the original item/keyFunc
//		b. The value inserted under that key is the timestamped item
//	2. Expiration happens lazily on read based on the expiration policy
//      a. No item can be inserted into the store while we're expiring
//		   *any* item in the cache.
//	3. Time-stamps are stripped off unexpired entries before return
// Note that the ExpirationCache is inherently slower than a normal
// threadSafeStore because it takes a write lock every time it checks if
// an item has expired.
type ExpirationCache struct {
	cacheStorage     ThreadSafeStore
	keyFunc          KeyFunc
	clock            clock.Clock
	expirationPolicy ExpirationPolicy
	// expirationLock is a write lock used to guarantee that we don't clobber
	// newly inserted objects because of a stale expiration timestamp comparison
	expirationLock sync.Mutex
}

// ExpirationPolicy dictates when an object expires. Currently only abstracted out
// so unittests don't rely on the system clock.
type ExpirationPolicy interface {
	IsExpired(obj *timestampedEntry) bool
}

// TTLPolicy implements a ttl based ExpirationPolicy.
type TTLPolicy struct {
	//	 >0: Expire entries with an age > ttl
	//	<=0: Don't expire any entry
	Ttl time.Duration

	// Clock used to calculate ttl expiration
	Clock clock.Clock
}

// IsExpired returns true if the given object is older than the ttl, or it can't
// determine its age.
func (p *TTLPolicy) IsExpired(obj *timestampedEntry) bool {
	return p.Ttl > 0 && p.Clock.Since(obj.timestamp) > p.Ttl
}

// timestampedEntry is the only type allowed in a ExpirationCache.
type timestampedEntry struct {
	obj       interface{}
	timestamp time.Time
}

// getTimestampedEntry returns the timestampedEntry stored under the given key.
func (c *ExpirationCache) getTimestampedEntry(key string) (*timestampedEntry, bool) {
	item, _ := c.cacheStorage.Get(key)
	if tsEntry, ok := item.(*timestampedEntry); ok {
		return tsEntry, true
	}
	return nil, false
}

// getOrExpire retrieves the object from the timestampedEntry if and only if it hasn't
// already expired. It holds a write lock across deletion.
func (c *ExpirationCache) getOrExpire(key string) (interface{}, bool) {
	// Prevent all inserts from the time we deem an item as "expired" to when we
	// delete it, so an un-expired item doesn't sneak in under the same key, just
	// before the Delete.
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	timestampedItem, exists := c.getTimestampedEntry(key)
	if !exists {
		return nil, false
	}
	if c.expirationPolicy.IsExpired(timestampedItem) {
		klog.V(4).Infof("Entry %v: %+v has expired", key, timestampedItem.obj)
		c.cacheStorage.Delete(key)
		return nil, false
	}
	return timestampedItem.obj, true
}

// GetByKey returns the item stored under the key, or sets exists=false.
func (c *ExpirationCache) GetByKey(key string) (interface{}, bool, error) {
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// Get returns unexpired items. It purges the cache of expired items in the
// process.
func (c *ExpirationCache) Get(obj interface{}) (interface{}, bool, error) {
	key, err := c.keyFunc(obj)
	if err != nil {
		return nil, false, KeyError{obj, err}
	}
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// List retrieves a list of unexpired items. It purges the cache of expired
// items in the process.
func (c *ExpirationCache) List() []interface{} {
	items := c.cacheStorage.List()

	list := make([]interface{}, 0, len(items))
	for _, item := range items {
		obj := item.(*timestampedEntry).obj
		if key, err := c.keyFunc(obj); err != nil {
			list = append(list, obj)
		} else if obj, exists := c.getOrExpire(key); exists {
			list = append(list, obj)
		}
	}
	return list
}

// ListKeys returns a list of all keys in the expiration cache.
func (c *ExpirationCache) ListKeys() []string {
	return c.cacheStorage.ListKeys()
}

// Add timestamps an item and inserts it into the cache, overwriting entries
// that might exist under the same key.
func (c *ExpirationCache) Add(obj interface{}) error {
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()

	c.cacheStorage.Add(key, &timestampedEntry{obj, c.clock.Now()})
	return nil
}

// Update has not been implemented yet for lack of a use case, so this method
// simply calls `Add`. This effectively refreshes the timestamp.
func (c *ExpirationCache) Update(obj interface{}) error {
	return c.Add(obj)
}

// Delete removes an item from the cache.
func (c *ExpirationCache) Delete(obj interface{}) error {
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	c.cacheStorage.Delete(key)
	return nil
}

// Replace will convert all items in the given list to TimestampedEntries
// before attempting the replace operation. The replace operation will
// delete the contents of the ExpirationCache `c`.
func (c *ExpirationCache) Replace(list []interface{}, resourceVersion string) error {
	items := make(map[string]interface{}, len(list))
	ts := c.clock.Now()
	for _, item := range list {
		key, err := c.keyFunc(item)
		if err != nil {
			return KeyError{item, err}
		}
		items[key] = &timestampedEntry{item, ts}
	}
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	c.cacheStorage.Replace(items, resourceVersion)
	return nil
}

// Resync will touch all objects to put them into the processing queue
func (c *ExpirationCache) Resync() error {
	return c.cacheStorage.Resync()
}

// NewTTLStore creates and returns a ExpirationCache with a TTLPolicy
func NewTTLStore(keyFunc KeyFunc, ttl time.Duration) Store {
	return &ExpirationCache{
		cacheStorage:     NewThreadSafeStore(Indexers{}, Indices{}),
		keyFunc:          keyFunc,
		clock:            clock.RealClock{},
		expirationPolicy: &TTLPolicy{ttl, clock.RealClock{}},
	}
}
<|endoftext|>