<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dhcp implements a DHCP client and server as described in RFC 2131.
package dhcp

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"time"

	"github.com/google/netstack/tcpip"
)

// Config is standard DHCP configuration.
type Config struct {
	Error         error
	ServerAddress tcpip.Address     // address of the server
	SubnetMask    tcpip.AddressMask // client address subnet mask
	Gateway       tcpip.Address     // client default gateway
	DNS           []tcpip.Address   // client DNS server addresses
	LeaseLength   time.Duration     // length of the address lease
}

func (cfg *Config) decode(opts []option) error {
	*cfg = Config{}
	for _, opt := range opts {
		b := opt.body
		if !opt.code.lenValid(len(b)) {
			// TODO: s/%v/%s/ when `go vet` is smarter.
			return fmt.Errorf("%v: bad length: %d", opt.code, len(b))
		}
		switch opt.code {
		case optLeaseTime:
			t := binary.BigEndian.Uint32(b)
			cfg.LeaseLength = time.Duration(t) * time.Second
		case optSubnetMask:
			cfg.SubnetMask = tcpip.AddressMask(b)
		case optDHCPServer:
			cfg.ServerAddress = tcpip.Address(b)
		case optDefaultGateway:
			cfg.Gateway = tcpip.Address(b)
		case optDomainNameServer:
			for ; len(b) > 0; b = b[4:] {
				if len(b) < 4 {
					return fmt.Errorf("DNS bad length: %d", len(b))
				}
				cfg.DNS = append(cfg.DNS, tcpip.Address(b[:4]))
			}
		}
	}
	return nil
}

func (cfg Config) encode() (opts []option) {
	if cfg.ServerAddress != "" {
		opts = append(opts, option{optDHCPServer, []byte(cfg.ServerAddress)})
	}
	if cfg.SubnetMask != "" {
		opts = append(opts, option{optSubnetMask, []byte(cfg.SubnetMask)})
	}
	if cfg.Gateway != "" {
		opts = append(opts, option{optDefaultGateway, []byte(cfg.Gateway)})
	}
	if len(cfg.DNS) > 0 {
		dns := make([]byte, 0, 4*len(cfg.DNS))
		for _, addr := range cfg.DNS {
			dns = append(dns, addr...)
		}
		opts = append(opts, option{optDomainNameServer, dns})
	}
	if l := cfg.LeaseLength / time.Second; l != 0 {
		v := make([]byte, 4)
		v[0] = byte(l >> 24)
		v[1] = byte(l >> 16)
		v[2] = byte(l >> 8)
		v[3] = byte(l >> 0)
		opts = append(opts, option{optLeaseTime, v})
	}
	return opts
}

const (
	// ServerPort is the well-known UDP port number for a DHCP server.
	ServerPort = 67
	// ClientPort is the well-known UDP port number for a DHCP client.
	ClientPort = 68
)

var magicCookie = []byte{99, 130, 83, 99} // RFC 1497

type xid uint32

type header []byte

func (h header) init() {
	h[1] = 0x01       // htype
	h[2] = 0x06       // hlen
	h[3] = 0x00       // hops
	h[8], h[9] = 0, 0 // secs
	copy(h[236:240], magicCookie)
}

func (h header) isValid() bool {
	if len(h) < 241 {
		return false
	}
	if o := h.op(); o != opRequest && o != opReply {
		return false
	}
	if h[1] != 0x01 || h[2] != 0x06 {
		return false
	}
	return bytes.Equal(h[236:240], magicCookie)
}

func (h header) op() op           { return op(h[0]) }
func (h header) setOp(o op)       { h[0] = byte(o) }
func (h header) xidbytes() []byte { return h[4:8] }
func (h header) xid() xid         { return xid(h[4])<<24 | xid(h[5])<<16 | xid(h[6])<<8 | xid(h[7]) }
func (h header) setBroadcast()    { h[10], h[11] = 0x80, 0x00 } // flags top bit
func (h header) ciaddr() []byte   { return h[12:16] }
func (h header) yiaddr() []byte   { return h[16:20] }
func (h header) siaddr() []byte   { return h[20:24] }
func (h header) giaddr() []byte   { return h[24:28] }
func (h header) chaddr() []byte   { return h[28:44] }
func (h header) sname() []byte    { return h[44:108] }
func (h header) file() []byte     { return h[108:236] }

func (h header) options() (opts options, err error) {
	i := headerBaseSize
	for i < len(h) {
		if h[i] == 0 {
			i++
			continue
		}
		if h[i] == 255 {
			break
		}
		if len(h) <= i+1 {
			return nil, fmt.Errorf("option missing length")
		}
		optlen := int(h[i+1])
		if len(h) < i+2+optlen {
			return nil, fmt.Errorf("option %v too long i=%d, optlen=%d", optionCode(h[i]), i, optlen)
		}
		opts = append(opts, option{
			code: optionCode(h[i]),
			body: h[i+2 : i+2+optlen],
		})
		i += 2 + optlen
	}
	return opts, nil
}

func (h header) setOptions(opts []option) {
	i := headerBaseSize
	for _, opt := range opts {
		h[i] = byte(opt.code)
		h[i+1] = byte(len(opt.body))
		copy(h[i+2:i+2+len(opt.body)], opt.body)
		i += 2 + len(opt.body)
	}
	h[i] = 255 // End option
	i++
	for ; i < len(h); i++ {
		h[i] = 0
	}
}

// headerBaseSize is the size of a DHCP packet, including the magic cookie.
//
// Note that a DHCP packet is required to have an 'end' option that takes
// up an extra byte, so the minimum DHCP packet size is headerBaseSize + 1.
const headerBaseSize = 240

type option struct {
	code optionCode
	body []byte
}

type optionCode byte

const (
	optSubnetMask       optionCode = 1
	optDefaultGateway   optionCode = 3
	optDomainNameServer optionCode = 6
	optDomainName       optionCode = 15
	optReqIPAddr        optionCode = 50
	optLeaseTime        optionCode = 51
	optDHCPMsgType      optionCode = 53 // dhcpMsgType
	optDHCPServer       optionCode = 54
	optParamReq         optionCode = 55
	optMessage          optionCode = 56
	optClientID         optionCode = 61
)

func (code optionCode) lenValid(l int) bool {
	switch code {
	case optSubnetMask, optDefaultGateway,
		optReqIPAddr, optLeaseTime, optDHCPServer:
		return l == 4
	case optDHCPMsgType:
		return l == 1
	case optDomainNameServer:
		return l%4 == 0
	case optMessage, optDomainName, optClientID:
		return l >= 1
	case optParamReq:
		return true // no fixed length
	default:
		return true // unknown option, assume ok
	}
}

type options []option

func (opts options) dhcpMsgType() (dhcpMsgType, error) {
	for _, opt := range opts {
		if opt.code == optDHCPMsgType {
			if len(opt.body) != 1 {
				// TODO: s/%v/%s/ when `go vet` is smarter.
				return 0, fmt.Errorf("%v: bad length: %d", opt.code, len(opt.body))
			}
			v := opt.body[0]
			if v <= 0 || v >= 8 {
				return 0, fmt.Errorf("DHCP bad length: %d", len(opt.body))
			}
			return dhcpMsgType(v), nil
		}
	}
	return 0, nil
}

func (opts options) message() string {
	for _, opt := range opts {
		if opt.code == optMessage {
			return string(opt.body)
		}
	}
	return ""
}

func (opts options) len() int {
	l := 0
	for _, opt := range opts {
		l += 1 + 1 + len(opt.body) // code + len + body
	}
	return l + 1 // extra byte for 'pad' option
}

type op byte

const (
	opRequest op = 0x01
	opReply   op = 0x02
)

// dhcpMsgType is the DHCP Message Type from RFC 1533, section 9.4.
type dhcpMsgType byte

const (
	dhcpDISCOVER dhcpMsgType = 1
	dhcpOFFER    dhcpMsgType = 2
	dhcpREQUEST  dhcpMsgType = 3
	dhcpDECLINE  dhcpMsgType = 4
	dhcpACK      dhcpMsgType = 5
	dhcpNAK      dhcpMsgType = 6
	dhcpRELEASE  dhcpMsgType = 7
)
<commit_msg>Resolve stringer TODO<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dhcp implements a DHCP client and server as described in RFC 2131.
package dhcp

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"time"

	"github.com/google/netstack/tcpip"
)

// Config is standard DHCP configuration.
type Config struct {
	Error         error
	ServerAddress tcpip.Address     // address of the server
	SubnetMask    tcpip.AddressMask // client address subnet mask
	Gateway       tcpip.Address     // client default gateway
	DNS           []tcpip.Address   // client DNS server addresses
	LeaseLength   time.Duration     // length of the address lease
}

func (cfg *Config) decode(opts []option) error {
	*cfg = Config{}
	for _, opt := range opts {
		b := opt.body
		if !opt.code.lenValid(len(b)) {
			return fmt.Errorf("%s: bad length: %d", opt.code, len(b))
		}
		switch opt.code {
		case optLeaseTime:
			t := binary.BigEndian.Uint32(b)
			cfg.LeaseLength = time.Duration(t) * time.Second
		case optSubnetMask:
			cfg.SubnetMask = tcpip.AddressMask(b)
		case optDHCPServer:
			cfg.ServerAddress = tcpip.Address(b)
		case optDefaultGateway:
			cfg.Gateway = tcpip.Address(b)
		case optDomainNameServer:
			for ; len(b) > 0; b = b[4:] {
				if len(b) < 4 {
					return fmt.Errorf("DNS bad length: %d", len(b))
				}
				cfg.DNS = append(cfg.DNS, tcpip.Address(b[:4]))
			}
		}
	}
	return nil
}

func (cfg Config) encode() (opts []option) {
	if cfg.ServerAddress != "" {
		opts = append(opts, option{optDHCPServer, []byte(cfg.ServerAddress)})
	}
	if cfg.SubnetMask != "" {
		opts = append(opts, option{optSubnetMask, []byte(cfg.SubnetMask)})
	}
	if cfg.Gateway != "" {
		opts = append(opts, option{optDefaultGateway, []byte(cfg.Gateway)})
	}
	if len(cfg.DNS) > 0 {
		dns := make([]byte, 0, 4*len(cfg.DNS))
		for _, addr := range cfg.DNS {
			dns = append(dns, addr...)
		}
		opts = append(opts, option{optDomainNameServer, dns})
	}
	if l := cfg.LeaseLength / time.Second; l != 0 {
		v := make([]byte, 4)
		v[0] = byte(l >> 24)
		v[1] = byte(l >> 16)
		v[2] = byte(l >> 8)
		v[3] = byte(l >> 0)
		opts = append(opts, option{optLeaseTime, v})
	}
	return opts
}

const (
	// ServerPort is the well-known UDP port number for a DHCP server.
	ServerPort = 67
	// ClientPort is the well-known UDP port number for a DHCP client.
	ClientPort = 68
)

var magicCookie = []byte{99, 130, 83, 99} // RFC 1497

type xid uint32

type header []byte

func (h header) init() {
	h[1] = 0x01       // htype
	h[2] = 0x06       // hlen
	h[3] = 0x00       // hops
	h[8], h[9] = 0, 0 // secs
	copy(h[236:240], magicCookie)
}

func (h header) isValid() bool {
	if len(h) < 241 {
		return false
	}
	if o := h.op(); o != opRequest && o != opReply {
		return false
	}
	if h[1] != 0x01 || h[2] != 0x06 {
		return false
	}
	return bytes.Equal(h[236:240], magicCookie)
}

func (h header) op() op           { return op(h[0]) }
func (h header) setOp(o op)       { h[0] = byte(o) }
func (h header) xidbytes() []byte { return h[4:8] }
func (h header) xid() xid         { return xid(h[4])<<24 | xid(h[5])<<16 | xid(h[6])<<8 | xid(h[7]) }
func (h header) setBroadcast()    { h[10], h[11] = 0x80, 0x00 } // flags top bit
func (h header) ciaddr() []byte   { return h[12:16] }
func (h header) yiaddr() []byte   { return h[16:20] }
func (h header) siaddr() []byte   { return h[20:24] }
func (h header) giaddr() []byte   { return h[24:28] }
func (h header) chaddr() []byte   { return h[28:44] }
func (h header) sname() []byte    { return h[44:108] }
func (h header) file() []byte     { return h[108:236] }

func (h header) options() (opts options, err error) {
	i := headerBaseSize
	for i < len(h) {
		if h[i] == 0 {
			i++
			continue
		}
		if h[i] == 255 {
			break
		}
		if len(h) <= i+1 {
			return nil, fmt.Errorf("option missing length")
		}
		optlen := int(h[i+1])
		if len(h) < i+2+optlen {
			return nil, fmt.Errorf("option %v too long i=%d, optlen=%d", optionCode(h[i]), i, optlen)
		}
		opts = append(opts, option{
			code: optionCode(h[i]),
			body: h[i+2 : i+2+optlen],
		})
		i += 2 + optlen
	}
	return opts, nil
}

func (h header) setOptions(opts []option) {
	i := headerBaseSize
	for _, opt := range opts {
		h[i] = byte(opt.code)
		h[i+1] = byte(len(opt.body))
		copy(h[i+2:i+2+len(opt.body)], opt.body)
		i += 2 + len(opt.body)
	}
	h[i] = 255 // End option
	i++
	for ; i < len(h); i++ {
		h[i] = 0
	}
}

// headerBaseSize is the size of a DHCP packet, including the magic cookie.
//
// Note that a DHCP packet is required to have an 'end' option that takes
// up an extra byte, so the minimum DHCP packet size is headerBaseSize + 1.
const headerBaseSize = 240

type option struct {
	code optionCode
	body []byte
}

type optionCode byte

const (
	optSubnetMask       optionCode = 1
	optDefaultGateway   optionCode = 3
	optDomainNameServer optionCode = 6
	optDomainName       optionCode = 15
	optReqIPAddr        optionCode = 50
	optLeaseTime        optionCode = 51
	optDHCPMsgType      optionCode = 53 // dhcpMsgType
	optDHCPServer       optionCode = 54
	optParamReq         optionCode = 55
	optMessage          optionCode = 56
	optClientID         optionCode = 61
)

func (code optionCode) lenValid(l int) bool {
	switch code {
	case optSubnetMask, optDefaultGateway,
		optReqIPAddr, optLeaseTime, optDHCPServer:
		return l == 4
	case optDHCPMsgType:
		return l == 1
	case optDomainNameServer:
		return l%4 == 0
	case optMessage, optDomainName, optClientID:
		return l >= 1
	case optParamReq:
		return true // no fixed length
	default:
		return true // unknown option, assume ok
	}
}

type options []option

func (opts options) dhcpMsgType() (dhcpMsgType, error) {
	for _, opt := range opts {
		if opt.code == optDHCPMsgType {
			if len(opt.body) != 1 {
				return 0, fmt.Errorf("%s: bad length: %d", opt.code, len(opt.body))
			}
			v := opt.body[0]
			if v <= 0 || v >= 8 {
				return 0, fmt.Errorf("DHCP bad length: %d", len(opt.body))
			}
			return dhcpMsgType(v), nil
		}
	}
	return 0, nil
}

func (opts options) message() string {
	for _, opt := range opts {
		if opt.code == optMessage {
			return string(opt.body)
		}
	}
	return ""
}

func (opts options) len() int {
	l := 0
	for _, opt := range opts {
		l += 1 + 1 + len(opt.body) // code + len + body
	}
	return l + 1 // extra byte for 'pad' option
}

type op byte

const (
	opRequest op = 0x01
	opReply   op = 0x02
)

// dhcpMsgType is the DHCP Message Type from RFC 1533, section 9.4.
type dhcpMsgType byte

const (
	dhcpDISCOVER dhcpMsgType = 1
	dhcpOFFER    dhcpMsgType = 2
	dhcpREQUEST  dhcpMsgType = 3
	dhcpDECLINE  dhcpMsgType = 4
	dhcpACK      dhcpMsgType = 5
	dhcpNAK      dhcpMsgType = 6
	dhcpRELEASE  dhcpMsgType = 7
)
<|endoftext|>


<commit_before>package diff

import (
	"reflect"
)

type Type int

const (
	TypesDiffer Type = iota
	ContentDiffer
	Identical
)

type Differ interface {
	Diff() Type
	Strings() []string
	StringIndent(key, prefix string, conf Output) string
}

type Comparaison struct {
	Type
	LHS Differ
	RHS Differ
}

func Diff(lhs, rhs interface{}) (Differ, error) {
	lhsVal := reflect.ValueOf(lhs)
	rhsVal := reflect.ValueOf(rhs)

	if lhs == nil && rhs == nil {
		return &Scalar{lhs, rhs}, nil
	}
	if lhs == nil || rhs == nil {
		return &Types{lhs, rhs}, nil
	}
	if lhsVal.Type().Comparable() && rhsVal.Type().Comparable() {
		return &Scalar{lhs, rhs}, nil
	}
	if lhsVal.Kind() != rhsVal.Kind() {
		return &Types{lhs, rhs}, nil
	}
	if lhsVal.Kind() == reflect.Slice {
		return NewSlice(lhs, rhs)
	}
	if lhsVal.Kind() == reflect.Map {
		return NewMap(lhs, rhs)
	}

	return &Types{lhs, rhs}, &Unsupported{lhsVal.Type(), rhsVal.Type()}
}

func (t Type) String() string {
	switch t {
	case Identical:
		return "identical"
	case ContentDiffer:
		return "content differ"
	case TypesDiffer:
		return "types differ"
	}

	return "invalid type"
}
<commit_msg>removing unused struct<commit_after>package diff

import (
	"reflect"
)

type Type int

const (
	TypesDiffer Type = iota
	ContentDiffer
	Identical
)

type Differ interface {
	Diff() Type
	Strings() []string
	StringIndent(key, prefix string, conf Output) string
}

func Diff(lhs, rhs interface{}) (Differ, error) {
	lhsVal := reflect.ValueOf(lhs)
	rhsVal := reflect.ValueOf(rhs)

	if lhs == nil && rhs == nil {
		return &Scalar{lhs, rhs}, nil
	}
	if lhs == nil || rhs == nil {
		return &Types{lhs, rhs}, nil
	}
	if lhsVal.Type().Comparable() && rhsVal.Type().Comparable() {
		return &Scalar{lhs, rhs}, nil
	}
	if lhsVal.Kind() != rhsVal.Kind() {
		return &Types{lhs, rhs}, nil
	}
	if lhsVal.Kind() == reflect.Slice {
		return NewSlice(lhs, rhs)
	}
	if lhsVal.Kind() == reflect.Map {
		return NewMap(lhs, rhs)
	}

	return &Types{lhs, rhs}, &Unsupported{lhsVal.Type(), rhsVal.Type()}
}

func (t Type) String() string {
	switch t {
	case Identical:
		return "identical"
	case ContentDiffer:
		return "content differ"
	case TypesDiffer:
		return "types differ"
	}

	return "invalid type"
}
<|endoftext|>


<commit_before>//Package anaconda provides structs and functions for accessing version 1.1
//of the Twitter API.
//
//Successful API queries return native Go structs that can be used immediately,
//with no need for type assertions.
//
//Authentication
//
//If you already have the access token (and secret) for your user (Twitter provides this for your own account on the developer portal), creating the client is simple:
//
//  anaconda.SetConsumerKey("your-consumer-key")
//  anaconda.SetConsumerSecret("your-consumer-secret")
//  api := anaconda.NewTwitterApi("your-access-token", "your-access-token-secret")
//
//
//Queries
//
//Executing queries on an authenticated TwitterApi struct is simple.
//
//  searchResult, _ := api.GetSearch("golang", nil)
//  for _ , tweet := range searchResult.Statuses {
//      fmt.Print(tweet.Text)
//  }
//
//Certain endpoints allow separate optional parameter; if desired, these can be passed as the final parameter.
//
//  v := url.Values{}
//  v.Set("count", "30")
//  result, err := api.GetSearch("golang", v)
//
//
//Endpoints
//
//Anaconda implements most of the endpoints defined in the Twitter API documentation: https://dev.twitter.com/docs/api/1.1.
//For clarity, in most cases, the function name is simply the name of the HTTP method and the endpoint (e.g., the endpoint `GET /friendships/incoming` is provided by the function `GetFriendshipsIncoming`).
//
//In a few cases, a shortened form has been chosen to make life easier (for example, retweeting is simply the function `Retweet`)
//
//More detailed information about the behavior of each particular endpoint can be found at the official Twitter API documentation.
package anaconda

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ChimeraCoder/tokenbucket"
	"github.com/garyburd/go-oauth/oauth"
)

const (
	_GET          = iota
	_POST         = iota
	BaseUrlV1     = "https://api.twitter.com/1"
	BaseUrl       = "https://api.twitter.com/1.1"
	UploadBaseUrl = "https://upload.twitter.com/1.1"
)

var oauthClient = oauth.Client{
	TemporaryCredentialRequestURI: "https://api.twitter.com/oauth/request_token",
	ResourceOwnerAuthorizationURI: "https://api.twitter.com/oauth/authenticate",
	TokenRequestURI:               "https://api.twitter.com/oauth/access_token",
}

type TwitterApi struct {
	Credentials          *oauth.Credentials
	queryQueue           chan query
	bucket               *tokenbucket.Bucket
	returnRateLimitError bool
	HttpClient           *http.Client

	// Currently used only for the streaming API
	// and for checking rate-limiting headers
	// Default logger is silent
	Log Logger

	// used for testing
	// defaults to BaseUrl
	baseUrl string
}

type query struct {
	url         string
	form        url.Values
	data        interface{}
	method      int
	response_ch chan response
}

type response struct {
	data interface{}
	err  error
}

const DEFAULT_DELAY = 0 * time.Second
const DEFAULT_CAPACITY = 5

//NewTwitterApi takes an user-specific access token and secret and returns a TwitterApi struct for that user.
//The TwitterApi struct can be used for accessing any of the endpoints available.
func NewTwitterApi(access_token string, access_token_secret string) *TwitterApi {
	//TODO figure out how much to buffer this channel
	//A non-buffered channel will cause blocking when multiple queries are made at the same time
	queue := make(chan query)
	c := &TwitterApi{
		Credentials: &oauth.Credentials{
			Token:  access_token,
			Secret: access_token_secret,
		},
		queryQueue:           queue,
		bucket:               nil,
		returnRateLimitError: false,
		HttpClient:           http.DefaultClient,
		Log:                  silentLogger{},
		baseUrl:              BaseUrl,
	}
	go c.throttledQuery()
	return c
}

//SetConsumerKey will set the application-specific consumer_key used in the initial OAuth process
//This key is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerKey(consumer_key string) {
	oauthClient.Credentials.Token = consumer_key
}

//SetConsumerSecret will set the application-specific secret used in the initial OAuth process
//This secret is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerSecret(consumer_secret string) {
	oauthClient.Credentials.Secret = consumer_secret
}

// ReturnRateLimitError specifies behavior when the Twitter API returns a rate-limit error.
// If set to true, the query will fail and return the error instead of automatically queuing and
// retrying the query when the rate limit expires
func (c *TwitterApi) ReturnRateLimitError(b bool) {
	c.returnRateLimitError = b
}

// Enable query throttling using the tokenbucket algorithm
func (c *TwitterApi) EnableThrottling(rate time.Duration, bufferSize int64) {
	c.bucket = tokenbucket.NewBucket(rate, bufferSize)
}

// Disable query throttling
func (c *TwitterApi) DisableThrottling() {
	c.bucket = nil
}

// SetDelay will set the delay between throttled queries
// To turn of throttling, set it to 0 seconds
func (c *TwitterApi) SetDelay(t time.Duration) {
	c.bucket.SetRate(t)
}

func (c *TwitterApi) GetDelay() time.Duration {
	return c.bucket.GetRate()
}

// SetBaseUrl is experimental and may be removed in future releases.
func (c *TwitterApi) SetBaseUrl(baseUrl string) {
	c.baseUrl = baseUrl
}

//AuthorizationURL generates the authorization URL for the first part of the OAuth handshake.
//Redirect the user to this URL.
//This assumes that the consumer key has already been set (using SetConsumerKey).
func AuthorizationURL(callback string) (string, *oauth.Credentials, error) {
	tempCred, err := oauthClient.RequestTemporaryCredentials(http.DefaultClient, callback, nil)
	if err != nil {
		return "", nil, err
	}
	return oauthClient.AuthorizationURL(tempCred, nil), tempCred, nil
}

func GetCredentials(tempCred *oauth.Credentials, verifier string) (*oauth.Credentials, url.Values, error) {
	return oauthClient.RequestToken(http.DefaultClient, tempCred, verifier)
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

// apiGet issues a GET request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiGet(urlStr string, form url.Values, data interface{}) error {
	filename := filepath.Join(append([]string{"json"}, strings.Split(strings.TrimPrefix(urlStr, c.baseUrl), "/")...)...)
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	resp, err := oauthClient.Get(c.HttpClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	resp.Body = ioutil.NopCloser(io.TeeReader(resp.Body, f))
	return decodeResponse(resp, data)
}

// apiPost issues a POST request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiPost(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Post(c.HttpClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// decodeResponse decodes the JSON response from the Twitter API.
func decodeResponse(resp *http.Response, data interface{}) error {
	if resp.StatusCode != 200 {
		return newApiError(resp)
	}
	return json.NewDecoder(resp.Body).Decode(data)
}

func NewApiError(resp *http.Response) *ApiError {
	body, _ := ioutil.ReadAll(resp.Body)

	return &ApiError{
		StatusCode: resp.StatusCode,
		Header:     resp.Header,
		Body:       string(body),
		URL:        resp.Request.URL,
	}
}

//query executes a query to the specified url, sending the values specified by form, and decodes the response JSON to data
//method can be either _GET or _POST
func (c TwitterApi) execQuery(urlStr string, form url.Values, data interface{}, method int) error {
	switch method {
	case _GET:
		return c.apiGet(urlStr, form, data)
	case _POST:
		return c.apiPost(urlStr, form, data)
	default:
		return fmt.Errorf("HTTP method not yet supported")
	}
}

// throttledQuery executes queries and automatically throttles them according to SECONDS_PER_QUERY
// It is the only function that reads from the queryQueue for a particular *TwitterApi struct

func (c *TwitterApi) throttledQuery() {
	for q := range c.queryQueue {
		url := q.url
		form := q.form
		data := q.data //This is where the actual response will be written
		method := q.method

		response_ch := q.response_ch

		if c.bucket != nil {
			<-c.bucket.SpendToken(1)
		}

		err := c.execQuery(url, form, data, method)

		// Check if Twitter returned a rate-limiting error
		if err != nil {
			if apiErr, ok := err.(*ApiError); ok {
				if isRateLimitError, nextWindow := apiErr.RateLimitCheck(); isRateLimitError && !c.returnRateLimitError {
					c.Log.Info(apiErr.Error())

					// If this is a rate-limiting error, re-add the job to the queue
					// TODO it really should preserve order
					go func() {
						c.queryQueue <- q
					}()

					delay := nextWindow.Sub(time.Now())
					<-time.After(delay)

					// Drain the bucket (start over fresh)
					if c.bucket != nil {
						c.bucket.Drain()
					}

					continue
				}
			}
		}

		response_ch <- response{data, err}
	}
}

// Close query queue
func (c *TwitterApi) Close() {
	close(c.queryQueue)
}
<commit_msg>Implement TeeWriter with dynamic filename<commit_after>//Package anaconda provides structs and functions for accessing version 1.1
//of the Twitter API.
//
//Successful API queries return native Go structs that can be used immediately,
//with no need for type assertions.
//
//Authentication
//
//If you already have the access token (and secret) for your user (Twitter provides this for your own account on the developer portal), creating the client is simple:
//
//  anaconda.SetConsumerKey("your-consumer-key")
//  anaconda.SetConsumerSecret("your-consumer-secret")
//  api := anaconda.NewTwitterApi("your-access-token", "your-access-token-secret")
//
//
//Queries
//
//Executing queries on an authenticated TwitterApi struct is simple.
//
//  searchResult, _ := api.GetSearch("golang", nil)
//  for _ , tweet := range searchResult.Statuses {
//      fmt.Print(tweet.Text)
//  }
//
//Certain endpoints allow separate optional parameter; if desired, these can be passed as the final parameter.
//
//  v := url.Values{}
//  v.Set("count", "30")
//  result, err := api.GetSearch("golang", v)
//
//
//Endpoints
//
//Anaconda implements most of the endpoints defined in the Twitter API documentation: https://dev.twitter.com/docs/api/1.1.
//For clarity, in most cases, the function name is simply the name of the HTTP method and the endpoint (e.g., the endpoint `GET /friendships/incoming` is provided by the function `GetFriendshipsIncoming`).
//
//In a few cases, a shortened form has been chosen to make life easier (for example, retweeting is simply the function `Retweet`)
//
//More detailed information about the behavior of each particular endpoint can be found at the official Twitter API documentation.
package anaconda

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ChimeraCoder/tokenbucket"
	"github.com/garyburd/go-oauth/oauth"
)

const (
	_GET          = iota
	_POST         = iota
	BaseUrlV1     = "https://api.twitter.com/1"
	BaseUrl       = "https://api.twitter.com/1.1"
	UploadBaseUrl = "https://upload.twitter.com/1.1"
)

var oauthClient = oauth.Client{
	TemporaryCredentialRequestURI: "https://api.twitter.com/oauth/request_token",
	ResourceOwnerAuthorizationURI: "https://api.twitter.com/oauth/authenticate",
	TokenRequestURI:               "https://api.twitter.com/oauth/access_token",
}

type TwitterApi struct {
	Credentials          *oauth.Credentials
	queryQueue           chan query
	bucket               *tokenbucket.Bucket
	returnRateLimitError bool
	HttpClient           *http.Client

	// Currently used only for the streaming API
	// and for checking rate-limiting headers
	// Default logger is silent
	Log Logger

	// used for testing
	// defaults to BaseUrl
	baseUrl string
}

type query struct {
	url         string
	form        url.Values
	data        interface{}
	method      int
	response_ch chan response
}

type response struct {
	data interface{}
	err  error
}

const DEFAULT_DELAY = 0 * time.Second
const DEFAULT_CAPACITY = 5

//NewTwitterApi takes an user-specific access token and secret and returns a TwitterApi struct for that user.
//The TwitterApi struct can be used for accessing any of the endpoints available.
func NewTwitterApi(access_token string, access_token_secret string) *TwitterApi {
	//TODO figure out how much to buffer this channel
	//A non-buffered channel will cause blocking when multiple queries are made at the same time
	queue := make(chan query)
	c := &TwitterApi{
		Credentials: &oauth.Credentials{
			Token:  access_token,
			Secret: access_token_secret,
		},
		queryQueue:           queue,
		bucket:               nil,
		returnRateLimitError: false,
		HttpClient:           http.DefaultClient,
		Log:                  silentLogger{},
		baseUrl:              BaseUrl,
	}
	go c.throttledQuery()
	return c
}

//SetConsumerKey will set the application-specific consumer_key used in the initial OAuth process
//This key is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerKey(consumer_key string) {
	oauthClient.Credentials.Token = consumer_key
}

//SetConsumerSecret will set the application-specific secret used in the initial OAuth process
//This secret is listed on https://dev.twitter.com/apps/YOUR_APP_ID/show
func SetConsumerSecret(consumer_secret string) {
	oauthClient.Credentials.Secret = consumer_secret
}

// ReturnRateLimitError specifies behavior when the Twitter API returns a rate-limit error.
// If set to true, the query will fail and return the error instead of automatically queuing and
// retrying the query when the rate limit expires
func (c *TwitterApi) ReturnRateLimitError(b bool) {
	c.returnRateLimitError = b
}

// Enable query throttling using the tokenbucket algorithm
func (c *TwitterApi) EnableThrottling(rate time.Duration, bufferSize int64) {
	c.bucket = tokenbucket.NewBucket(rate, bufferSize)
}

// Disable query throttling
func (c *TwitterApi) DisableThrottling() {
	c.bucket = nil
}

// SetDelay will set the delay between throttled queries
// To turn of throttling, set it to 0 seconds
func (c *TwitterApi) SetDelay(t time.Duration) {
	c.bucket.SetRate(t)
}

func (c *TwitterApi) GetDelay() time.Duration {
	return c.bucket.GetRate()
}

// SetBaseUrl is experimental and may be removed in future releases.
func (c *TwitterApi) SetBaseUrl(baseUrl string) {
	c.baseUrl = baseUrl
}

//AuthorizationURL generates the authorization URL for the first part of the OAuth handshake.
//Redirect the user to this URL.
//This assumes that the consumer key has already been set (using SetConsumerKey).
func AuthorizationURL(callback string) (string, *oauth.Credentials, error) {
	tempCred, err := oauthClient.RequestTemporaryCredentials(http.DefaultClient, callback, nil)
	if err != nil {
		return "", nil, err
	}
	return oauthClient.AuthorizationURL(tempCred, nil), tempCred, nil
}

func GetCredentials(tempCred *oauth.Credentials, verifier string) (*oauth.Credentials, url.Values, error) {
	return oauthClient.RequestToken(http.DefaultClient, tempCred, verifier)
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

// apiGet issues a GET request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiGet(urlStr string, form url.Values, data interface{}) error {
	var f io.WriteCloser

	if oauthClient.Credentials.Token != "" {
		filename := filepath.Join(append([]string{"json"}, strings.Split(strings.TrimPrefix(urlStr, c.baseUrl), "/")...)...)

		dirs := strings.TrimSuffix(filename, filepath.Base(filename))
		err := os.MkdirAll(dirs, 0755)
		if err != nil {
			return err
		}

		f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
		if err != nil {
			return err
		}
		defer f.Close()
	} else {
		f = os.Stdout
	}

	resp, err := oauthClient.Get(c.HttpClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	resp.Body = ioutil.NopCloser(io.TeeReader(resp.Body, f))
	return decodeResponse(resp, data)
}

// apiPost issues a POST request to the Twitter API and decodes the response JSON to data.
func (c TwitterApi) apiPost(urlStr string, form url.Values, data interface{}) error {
	resp, err := oauthClient.Post(c.HttpClient, c.Credentials, urlStr, form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return decodeResponse(resp, data)
}

// decodeResponse decodes the JSON response from the Twitter API.
func decodeResponse(resp *http.Response, data interface{}) error {
	if resp.StatusCode != 200 {
		return newApiError(resp)
	}
	return json.NewDecoder(resp.Body).Decode(data)
}

func NewApiError(resp *http.Response) *ApiError {
	body, _ := ioutil.ReadAll(resp.Body)

	return &ApiError{
		StatusCode: resp.StatusCode,
		Header:     resp.Header,
		Body:       string(body),
		URL:        resp.Request.URL,
	}
}

//query executes a query to the specified url, sending the values specified by form, and decodes the response JSON to data
//method can be either _GET or _POST
func (c TwitterApi) execQuery(urlStr string, form url.Values, data interface{}, method int) error {
	switch method {
	case _GET:
		return c.apiGet(urlStr, form, data)
	case _POST:
		return c.apiPost(urlStr, form, data)
	default:
		return fmt.Errorf("HTTP method not yet supported")
	}
}

// throttledQuery executes queries and automatically throttles them according to SECONDS_PER_QUERY
// It is the only function that reads from the queryQueue for a particular *TwitterApi struct

func (c *TwitterApi) throttledQuery() {
	for q := range c.queryQueue {
		url := q.url
		form := q.form
		data := q.data //This is where the actual response will be written
		method := q.method

		response_ch := q.response_ch

		if c.bucket != nil {
			<-c.bucket.SpendToken(1)
		}

		err := c.execQuery(url, form, data, method)

		// Check if Twitter returned a rate-limiting error
		if err != nil {
			if apiErr, ok := err.(*ApiError); ok {
				if isRateLimitError, nextWindow := apiErr.RateLimitCheck(); isRateLimitError && !c.returnRateLimitError {
					c.Log.Info(apiErr.Error())

					// If this is a rate-limiting error, re-add the job to the queue
					// TODO it really should preserve order
					go func() {
						c.queryQueue <- q
					}()

					delay := nextWindow.Sub(time.Now())
					<-time.After(delay)

					// Drain the bucket (start over fresh)
					if c.bucket != nil {
						c.bucket.Drain()
					}

					continue
				}
			}
		}

		response_ch <- response{data, err}
	}
}

// Close query queue
func (c *TwitterApi) Close() {
	close(c.queryQueue)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"os"
	"sync"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/backup"
	"github.com/jacobsa/comeback/internal/wiring"
)

var gDirSaverOnce sync.Once
var gDirSaver backup.DirectorySaver

func initDirSaver(ctx context.Context) {
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	bucket := getBucket(ctx)
	password := getPassword()
	state := getState(ctx)

	const chunkSize = 1 << 24 // 16 MiB
	gDirSaver, err = wiring.MakeDirSaver(
		ctx,
		password,
		bucket,
		chunkSize,
		state.ExistingScores,
		state.ScoresForFiles,
		log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds))
}

func getDirSaver(ctx context.Context) backup.DirectorySaver {
	gDirSaverOnce.Do(func() { initDirSaver(ctx) })
	return gDirSaver
}
<commit_msg>Deleted dir_saver.go.<commit_after><|endoftext|>


<commit_before>// Package disc is a LAN discovery library
package disc

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/olekukonko/tablewriter"

	"github.com/mehrdadrad/mylg/cli"
)

const (
	// IEEEOUI holds ieee oui csv file url
	IEEEOUI = "http://standards.ieee.org/develop/regauth/oui/oui.csv"
)

// ARP holds ARP information
type ARP struct {
	IP        string
	MAC       string
	Interface string
	Host      string
}

// disc holds all discovery information
type disc struct {
	Table []ARP
	IPs   []string
	OUI   map[string]string
	IsBSD bool
	SKey  string
}

// New creates new discovery object
func New(args string) *disc {
	key, flag := cli.Flag(args)

	// help
	if _, ok := flag["help"]; ok {
		help()
		return nil
	}

	return &disc{
		IsBSD: IsBSD(),
		OUI:   make(map[string]string, 25000),
		SKey:  key,
	}
}

// WalkIP tries to salk through subnet as generator
func WalkIP(cidr string) chan string {
	c := make(chan string, 1)
	go func() {
		defer close(c)
		ip, ipnet, err := net.ParseCIDR(cidr)
		if err != nil {
			println(err.Error())
			return
		}
		for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); nextIP(ip) {
			c <- ip.String()
		}
	}()
	return c
}

// PingLan tries to send a tiny UDP packet to all LAN hosts
func (a *disc) PingLan() {
	var isV4 bool
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP)
	if err != nil {
		println(err.Error())
		return
	}
	defer syscall.Close(fd)
	// Set options
	syscall.SetsockoptInt(fd, 0x0, syscall.IP_TTL, 10)
	ifs, _ := net.Interfaces()
	for _, i := range ifs {
		addrs, _ := i.Addrs()
		if i.Flags != 19 {
			continue
		}
		// ip network(s) that assigned to interface(s)
		for _, addr := range addrs {
			if strings.IndexAny(addr.String(), "::") != -1 {
				isV4 = false
			} else {
				isV4 = true
			}
			if isV4 {
				a.IPs = append(a.IPs, addr.String())
				for ipStr := range WalkIP(addr.String()) {
					ip := net.ParseIP(ipStr).To4()
					addr := syscall.SockaddrInet4{
						Port: 33434,
						Addr: [4]byte{ip[0], ip[1], ip[2], ip[3]},
					}
					syscall.Sendto(fd, []byte{}, 0, &addr)
				}
			} else {
				//IPv6 doesn't support for the time being
				// ToDo
			}
		}
	}
}
func nextIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// StrTobyte16 converts string to 16 bytes
func StrTobyte16(s string) [16]byte {
	var r [16]byte
	if len(s) > 16 {
		copy(r[:], s)
	} else {
		copy(r[16-len(s):], s)
	}
	return r
}

// GetARPTable gets ARP table
func (a *disc) GetARPTable() error {
	if a.IsBSD {
		return a.GetMACOSARPTable()
	}
	return a.GetLinuxARPTable()
}

// GetLinuxARPTable gets Linux ARP table
func (a *disc) GetLinuxARPTable() error {
	var host string
	file, err := os.Open("/proc/net/arp")
	if err != nil {
		return err
	}
	s := bufio.NewScanner(file)
	s.Scan() // skip description
	for s.Scan() {
		fields := strings.Fields(s.Text())
		hosts, _ := net.LookupAddr(fields[0])

		if len(hosts) == 0 {
			host = "NA"
		} else {
			host = hosts[0]
		}

		a.Table = append(a.Table, ARP{IP: fields[0], Host: host, MAC: fields[3], Interface: fields[5]})
	}
	return nil
}

// GetMACOSARPTable gets Mac OS X ARP table
func (a *disc) GetMACOSARPTable() error {
	var host string
	cmd := exec.Command("arp", "-an")
	outBytes, err := cmd.Output()
	if err != nil {
		return err
	}
	out := strings.TrimSpace(string(outBytes))
	for _, l := range strings.Split(out, "\n") {
		fields := strings.Fields(l)
		if len(fields) < 1 {
			continue
		}
		if fields[3] != "(incomplete)" {
			fields[1] = strings.Trim(fields[1], ")")
			fields[1] = strings.Trim(fields[1], "(")
			hosts, _ := net.LookupAddr(fields[1])

			if len(hosts) == 0 {
				host = "NA"
			} else {
				host = hosts[0]
			}

			a.Table = append(a.Table, ARP{IP: fields[1], Host: host, MAC: fields[3], Interface: fields[5]})
		}
	}
	return nil
}

// LoadOUI
func (a *disc) LoadOUI() bool {
	if _, ok := cache("validate", nil); ok {
		if c, ok := cache("read", nil); ok {
			r := csv.NewReader(strings.NewReader(c))
			records, _ := r.ReadAll()
			for _, record := range records {
				if len(record) > 2 {
					a.OUI[record[1]] = record[2]
				}
			}
			return true
		}

	} else {
		b, err := GetOUILive()
		if err != nil {
			println(err.Error())
			return false
		}
		if c, ok := cache("write", b); ok {
			r := csv.NewReader(strings.NewReader(c))
			records, _ := r.ReadAll()
			for _, record := range records {
				if len(record) > 2 {
					a.OUI[record[1]] = record[2]
				}
			}
			return true
		}
	}
	return false
}

// GetOUILive gets oui info from iEEE
func GetOUILive() ([]byte, error) {
	resp, err := http.Get(IEEEOUI)
	if err != nil {
		return []byte{}, fmt.Errorf("regauth.standards.ieee.org is unreachable (1)")
	}
	if resp.StatusCode != 200 {
		return []byte{}, fmt.Errorf("regauth.standards.ieee.org returns none 200 HTTP code")
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, fmt.Errorf("peeringdb.com is unreachable (2)  %s", err.Error())
	}
	return body, nil
}
func cache(r string, b []byte) (string, bool) {
	var (
		err error
		res string
	)
	switch r {
	case "write":
		err = ioutil.WriteFile("/tmp/mylg.disc", b, 0644)
		if err != nil {
			return "", false
		}
		res = string(b)
		return res, true
	case "read":
		b, err := ioutil.ReadFile("/tmp/mylg.disc")
		if err != nil {
			return "", false
		}
		res = string(b)
		return res, true
	case "validate":
		f, err := os.Stat("/tmp/mylg.disc")
		if err != nil {
			return "", false
		}
		d := time.Since(f.ModTime())
		if d.Hours() > 24*10 {
			return "", false
		}
	}

	return "", true
}

// PrintPretty prints ARP table
func (a *disc) PrintPretty() {
	var (
		orgName string
		counter = 0
	)
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"IP", "MAC", "Host", "Interface", "Organization Name"})
	for _, arp := range a.Table {

		// OUI
		if name, ok := a.OUI[strings.ToUpper(strings.Replace(arp.MAC, ":", "", -1))[:6]]; ok {
			if len(name) > 25 {
				orgName = name[:25] + "..."
			} else {
				orgName = name
			}
		} else {
			orgName = "NA"
		}

		// Search
		sHost := search(arp.Host, a.SKey)
		sARP := search(arp.IP, a.SKey)
		sMAC := search(arp.MAC, a.SKey)
		sOrg := search(orgName, a.SKey)

		if !sHost && !sARP && !sMAC && !sOrg {
			continue
		}

		table.Append([]string{arp.IP, arp.MAC, arp.Host, arp.Interface, orgName})
		counter++
	}
	table.Render()
	println(counter, "host has been found")
}

// IsBSD checks OS
func IsBSD() bool {
	if runtime.GOOS != "linux" {
		return true
	}
	return false
}

// search tries to find key at data
func search(data, key string) bool {
	data = strings.ToLower(data)
	key = strings.ToLower(key)
	if strings.Contains(data, key) {
		return true
	}
	return false
}

// help shows disc help
func help() {
	fmt.Println(`
    Network LAN Discovery
    usage:
          disc [search keyword]
    Example:
          disc
          disc 5c:a:5b:aa:4a:99
          disc apple
          disc 192.168.0.10
	`)
}
<commit_msg>minor fixes<commit_after>// Package disc is a LAN discovery library
package disc

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/olekukonko/tablewriter"

	"github.com/mehrdadrad/mylg/cli"
)

const (
	// IEEEOUI holds ieee oui csv file url
	IEEEOUI = "http://standards.ieee.org/develop/regauth/oui/oui.csv"
)

// ARP holds ARP information
type ARP struct {
	IP        string
	MAC       string
	Interface string
	Host      string
}

// disc holds all discovery information
type disc struct {
	Table []ARP
	IPs   []string
	OUI   map[string]string
	IsBSD bool
	SKey  string
}

// New creates new discovery object
func New(args string) *disc {
	key, flag := cli.Flag(args)

	// help
	if _, ok := flag["help"]; ok {
		help()
		return nil
	}

	return &disc{
		IsBSD: IsBSD(),
		OUI:   make(map[string]string, 25000),
		SKey:  key,
	}
}

// WalkIP tries to salk through subnet as generator
func WalkIP(cidr string) chan string {
	c := make(chan string, 1)
	go func() {
		defer close(c)
		ip, ipnet, err := net.ParseCIDR(cidr)
		if err != nil {
			println(err.Error())
			return
		}
		for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); nextIP(ip) {
			c <- ip.String()
		}
	}()
	return c
}

// PingLan tries to send a tiny UDP packet to all LAN hosts
func (a *disc) PingLan() {
	var isV4 bool
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP)
	if err != nil {
		println(err.Error())
		return
	}
	defer syscall.Close(fd)
	// Set options
	syscall.SetsockoptInt(fd, 0x0, syscall.IP_TTL, 10)
	ifs, _ := net.Interfaces()
	for _, i := range ifs {
		addrs, _ := i.Addrs()
		if i.Flags != 19 {
			continue
		}
		// ip network(s) that assigned to interface(s)
		for _, addr := range addrs {
			if strings.IndexAny(addr.String(), "::") != -1 {
				isV4 = false
			} else {
				isV4 = true
			}
			if isV4 {
				a.IPs = append(a.IPs, addr.String())
				for ipStr := range WalkIP(addr.String()) {
					ip := net.ParseIP(ipStr).To4()
					addr := syscall.SockaddrInet4{
						Port: 33434,
						Addr: [4]byte{ip[0], ip[1], ip[2], ip[3]},
					}
					syscall.Sendto(fd, []byte{}, 0, &addr)
				}
			} else {
				//IPv6 doesn't support for the time being
				// ToDo
			}
		}
	}
}
func nextIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// StrTobyte16 converts string to 16 bytes
func StrTobyte16(s string) [16]byte {
	var r [16]byte
	if len(s) > 16 {
		copy(r[:], s)
	} else {
		copy(r[16-len(s):], s)
	}
	return r
}

// GetARPTable gets ARP table
func (a *disc) GetARPTable() error {
	if a.IsBSD {
		return a.GetMACOSARPTable()
	}
	return a.GetLinuxARPTable()
}

// GetLinuxARPTable gets Linux ARP table
func (a *disc) GetLinuxARPTable() error {
	var host string
	file, err := os.Open("/proc/net/arp")
	if err != nil {
		return err
	}
	s := bufio.NewScanner(file)
	s.Scan() // skip description
	for s.Scan() {
		fields := strings.Fields(s.Text())
		hosts, _ := net.LookupAddr(fields[0])

		if len(hosts) == 0 {
			host = "NA"
		} else {
			host = hosts[0]
		}

		a.Table = append(a.Table, ARP{IP: fields[0], Host: host, MAC: fields[3], Interface: fields[5]})
	}
	return nil
}

// GetMACOSARPTable gets Mac OS X ARP table
func (a *disc) GetMACOSARPTable() error {
	var host string
	cmd := exec.Command("arp", "-an")
	outBytes, err := cmd.Output()
	if err != nil {
		return err
	}
	out := strings.TrimSpace(string(outBytes))
	for _, l := range strings.Split(out, "\n") {
		fields := strings.Fields(l)
		if len(fields) < 1 {
			continue
		}
		if fields[3] != "(incomplete)" {
			fields[1] = strings.Trim(fields[1], ")")
			fields[1] = strings.Trim(fields[1], "(")
			hosts, _ := net.LookupAddr(fields[1])

			if len(hosts) == 0 {
				host = "NA"
			} else {
				host = hosts[0]
			}

			a.Table = append(a.Table, ARP{IP: fields[1], Host: host, MAC: fields[3], Interface: fields[5]})
		}
	}
	return nil
}

// LoadOUI
func (a *disc) LoadOUI() bool {
	if _, ok := cache("validate", nil); ok {
		if c, ok := cache("read", nil); ok {
			r := csv.NewReader(strings.NewReader(c))
			records, _ := r.ReadAll()
			for _, record := range records {
				if len(record) > 2 {
					a.OUI[record[1]] = record[2]
				}
			}
			return true
		}

	} else {
		b, err := GetOUILive()
		if err != nil {
			println(err.Error())
			return false
		}
		if c, ok := cache("write", b); ok {
			r := csv.NewReader(strings.NewReader(c))
			records, _ := r.ReadAll()
			for _, record := range records {
				if len(record) > 2 {
					a.OUI[record[1]] = record[2]
				}
			}
			return true
		}
	}
	return false
}

// GetOUILive gets oui info from iEEE
func GetOUILive() ([]byte, error) {
	resp, err := http.Get(IEEEOUI)
	if err != nil {
		return []byte{}, fmt.Errorf("regauth.standards.ieee.org is unreachable (1)")
	}
	if resp.StatusCode != 200 {
		return []byte{}, fmt.Errorf("regauth.standards.ieee.org returns none 200 HTTP code")
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, fmt.Errorf("peeringdb.com is unreachable (2)  %s", err.Error())
	}
	return body, nil
}
func cache(r string, b []byte) (string, bool) {
	var (
		err error
		res string
	)
	switch r {
	case "write":
		err = ioutil.WriteFile("/tmp/mylg.disc", b, 0644)
		if err != nil {
			return "", false
		}
		res = string(b)
		return res, true
	case "read":
		b, err := ioutil.ReadFile("/tmp/mylg.disc")
		if err != nil {
			return "", false
		}
		res = string(b)
		return res, true
	case "validate":
		f, err := os.Stat("/tmp/mylg.disc")
		if err != nil {
			return "", false
		}
		d := time.Since(f.ModTime())
		if d.Hours() > 24*10 {
			return "", false
		}
	}

	return "", true
}

// PrintPretty prints ARP table
func (a *disc) PrintPretty() {
	var (
		orgName string
		counter = 0
	)
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"IP", "MAC", "Host", "Interface", "Organization Name"})
	for _, arp := range a.Table {

		// OUI
		if name, ok := a.OUI[strings.ToUpper(strings.Replace(arp.MAC, ":", "", -1))[:6]]; ok {
			if len(name) > 25 {
				orgName = name[:25] + "..."
			} else {
				orgName = name
			}
		} else {
			orgName = "NA"
		}

		// Search
		sHost := search(arp.Host, a.SKey)
		sARP := search(arp.IP, a.SKey)
		sMAC := search(arp.MAC, a.SKey)
		sOrg := search(orgName, a.SKey)

		if !sHost && !sARP && !sMAC && !sOrg {
			continue
		}

		table.Append([]string{arp.IP, arp.MAC, arp.Host, arp.Interface, orgName})
		counter++
	}
	table.Render()
	println(counter, "host(s) has been found")
}

// IsBSD checks OS
func IsBSD() bool {
	if runtime.GOOS != "linux" {
		return true
	}
	return false
}

// search tries to find key at data
func search(data, key string) bool {
	data = strings.ToLower(data)
	key = strings.ToLower(key)
	if strings.Contains(data, key) {
		return true
	}
	return false
}

// help shows disc help
func help() {
	fmt.Println(`
    Network LAN Discovery
    usage:
          disc [search keyword]
    Example:
          disc
          disc 5c:a:5b:aa:4a:99
          disc apple
          disc 192.168.0.10
	`)
}
<|endoftext|>


<commit_before>// Copyright 2010 The W32 Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package w32

import (
    "unsafe"
)

type (
    BOOL         int
    ATOM         uint16
    DWORD        uint16
    HANDLE       uintptr
    HINSTANCE    HANDLE
    HACCEL       HANDLE
    HCURSOR      HANDLE
    HDWP         HANDLE
    HICON        HANDLE
    HMENU        HANDLE
    HWND         HANDLE
    HBRUSH       HANDLE
    HRESULT      uint
    HFONT        HANDLE
    HDC          HANDLE
    HGDIOBJ      HANDLE
    HDROP        HANDLE
    HENHMETAFILE HANDLE
    HBITMAP      HANDLE
    HPEN         HANDLE
    HKEY         HANDLE
    HIMAGELIST   HANDLE
    HGLOBAL      HANDLE
    HMODULE      HANDLE
    HRSRC        HANDLE
    COLORREF     uint
)

type POINT struct {
    X, Y int
}

type RECT struct {
    Left, Top, Right, Bottom int
}

type WNDCLASSEX struct {
    Size       uint
    Style      uint
    WndProc    uintptr
    ClsExtra   int
    WndExtra   int
    Instance   HINSTANCE
    Icon       HICON
    Cursor     HCURSOR
    Background HBRUSH
    MenuName   *uint16
    ClassName  *uint16
    IconSm     HICON
}

type MSG struct {
    Hwnd    HWND
    Message uint
    WParam  uintptr
    LParam  uintptr
    Time    uint
    Pt      POINT
}

type LOGFONT struct {
    Height         int
    Width          int
    Escapement     int
    Orientation    int
    Weight         int
    Italic         byte
    Underline      byte
    StrikeOut      byte
    CharSet        byte
    OutPrecision   byte
    ClipPrecision  byte
    Quality        byte
    PitchAndFamily byte
    FaceName       [LF_FACESIZE]uint16
}

type OPENFILENAME struct {
    StructSize      uint
    Owner           HWND
    Instance        HINSTANCE
    Filter          *uint16
    CustomFilter    *uint16
    MaxCustomFilter uint
    FilterIndex     uint
    File            *uint16
    MaxFile         uint
    FileTitle       *uint16
    MaxFileTitle    uint
    InitialDir      *uint16
    Title           *uint16
    Flags           uint
    FileOffset      uint16
    FileExtension   uint16
    DefExt          *uint16
    CustData        uintptr
    FnHook          uintptr
    TemplateName    *uint16
    PvReserved      unsafe.Pointer
    DwReserved      uint
    FlagsEx         uint
}

type BROWSEINFO struct {
    Owner        HWND
    Root         *uint16
    DisplayName  *uint16
    Title        *uint16
    Flags        uint
    CallbackFunc uintptr
    LParam       uintptr
    Image        int
}

type GUID struct {
    Data1 uint
    Data2 uint16
    Data3 uint16
    Data4 [8]byte
}

type VARIANT struct {
    VT         uint16 //  2
    WReserved1 uint16 //  4
    WReserved2 uint16 //  6
    WReserved3 uint16 //  8
    Val        int64  // 16
}

type DISPPARAMS struct {
    Rgvarg            uintptr
    RgdispidNamedArgs uintptr
    CArgs             uint
    CNamedArgs        uint
}

type EXCEPINFO struct {
    WCode             uint16
    WReserved         uint16
    BstrSource        *uint16
    BstrDescription   *uint16
    BstrHelpFile      *uint16
    DwHelpContext     uint
    PvReserved        uintptr
    PfnDeferredFillIn uintptr
    Scode             int
}

type LOGBRUSH struct {
    LbStyle uint
    LbColor COLORREF
    LbHatch uintptr
}

type DEVMODE struct {
    DmDeviceName       [CCHDEVICENAME]uint16
    DmSpecVersion      uint16
    DmDriverVersion    uint16
    DmSize             uint16
    DmDriverExtra      uint16
    DmFields           uint
    DmOrientation      int16
    DmPaperSize        int16
    DmPaperLength      int16
    DmPaperWidth       int16
    DmScale            int16
    DmCopies           int16
    DmDefaultSource    int16
    DmPrintQuality     int16
    DmColor            int16
    DmDuplex           int16
    DmYResolution      int16
    DmTTOption         int16
    DmCollate          int16
    DmFormName         [CCHFORMNAME]uint16
    DmLogPixels        uint16
    DmBitsPerPel       uint
    DmPelsWidth        uint
    DmPelsHeight       uint
    DmDisplayFlags     uint
    DmDisplayFrequency uint
    DmICMMethod        uint
    DmICMIntent        uint
    DmMediaType        uint
    DmDitherType       uint
    DmReserved1        uint
    DmReserved2        uint
    DmPanningWidth     uint
    DmPanningHeight    uint
}

type BITMAPINFOHEADER struct {
    BiSize          uint
    BiWidth         int
    BiHeight        int
    BiPlanes        uint16
    BiBitCount      uint16
    BiCompression   uint
    BiSizeImage     uint
    BiXPelsPerMeter int
    BiYPelsPerMeter int
    BiClrUsed       uint
    BiClrImportant  uint
}

type RGBQUAD struct {
    RgbBlue     byte
    RgbGreen    byte
    RgbRed      byte
    RgbReserved byte
}

type BITMAPINFO struct {
    BmiHeader BITMAPINFOHEADER
    BmiColors *RGBQUAD
}

type BITMAP struct {
    BmType       int32
    BmWidth      int32
    BmHeight     int32
    BmWidthBytes int32
    BmPlanes     uint16
    BmBitsPixel  uint16
    BmBits       unsafe.Pointer
}

type DIBSECTION struct {
    DsBm        BITMAP
    DsBmih      BITMAPINFOHEADER
    DsBitfields [3]uint32
    DshSection  HANDLE
    DsOffset    uint32
}

type ENHMETAHEADER struct {
    IType          uint
    NSize          uint
    RclBounds      RECT
    RclFrame       RECT
    DSignature     uint
    NVersion       uint
    NBytes         uint
    NRecords       uint
    NHandles       uint16
    SReserved      uint16
    NDescription   uint
    OffDescription uint
    NPalEntries    uint
    SzlDevice      SIZE
    SzlMillimeters SIZE
    CbPixelFormat  uint
    OffPixelFormat uint
    BOpenGL        uint
    SzlMicrometers SIZE
}

type SIZE struct {
    CX, CY int
}

type TEXTMETRIC struct {
    TmHeight           int
    TmAscent           int
    TmDescent          int
    TmInternalLeading  int
    TmExternalLeading  int
    TmAveCharWidth     int
    TmMaxCharWidth     int
    TmWeight           int
    TmOverhang         int
    TmDigitizedAspectX int
    TmDigitizedAspectY int
    TmFirstChar        uint16
    TmLastChar         uint16
    TmDefaultChar      uint16
    TmBreakChar        uint16
    TmItalic           byte
    TmUnderlined       byte
    TmStruckOut        byte
    TmPitchAndFamily   byte
    TmCharSet          byte
}

type DOCINFO struct {
    CbSize       int
    LpszDocName  *uint16
    LpszOutput   *uint16
    LpszDatatype *uint16
    FwType       uint
}

type NMHDR struct {
    HwndFrom HWND
    IdFrom   uintptr
    Code     uint
}

type LVCOLUMN struct {
    Mask       uint
    Fmt        int
    Cx         int
    PszText    *uint16
    CchTextMax int
    ISubItem   int
    IImage     int
    IOrder     int
}

type LVITEM struct {
    Mask       uint
    IItem      int
    ISubItem   int
    State      uint
    StateMask  uint
    PszText    *uint16
    CchTextMax int
    IImage     int
    LParam     uintptr
    IIndent    int
    IGroupId   int
    CColumns   uint
    PuColumns  uint
}

type LVHITTESTINFO struct {
    Pt       POINT
    Flags    uint
    IItem    int
    ISubItem int
    IGroup   int
}

type NMITEMACTIVATE struct {
    Hdr       NMHDR
    IItem     int
    ISubItem  int
    UNewState uint
    UOldState uint
    UChanged  uint
    PtAction  POINT
    LParam    uintptr
    UKeyFlags uint
}

type NMLISTVIEW struct {
    Hdr       NMHDR
    IItem     int
    ISubItem  int
    UNewState uint
    UOldState uint
    UChanged  uint
    PtAction  POINT
    LParam    uintptr
}

type NMLVDISPINFO struct {
    Hdr  NMHDR
    Item LVITEM
}

type INITCOMMONCONTROLSEX struct {
    DwSize uint
    DwICC  uint
}

type TOOLINFO struct {
    CbSize     uint
    UFlags     uint
    Hwnd       HWND
    UId        uintptr
    Rect       RECT
    Hinst      HINSTANCE
    LpszText   *uint16
    LParam     uintptr
    LpReserved unsafe.Pointer
}

type TRACKMOUSEEVENT struct {
    CbSize      uint
    DwFlags     uint
    HwndTrack   HWND
    DwHoverTime uint
}

type GdiplusStartupInput struct {
    GdiplusVersion           uint32
    DebugEventCallback       uintptr
    SuppressBackgroundThread BOOL
    SuppressExternalCodecs   BOOL
}

type GdiplusStartupOutput struct {
    NotificationHook   uintptr
    NotificationUnhook uintptr
}

type PAINTSTRUCT struct {
    Hdc         HDC
    FErase      BOOL
    RcPaint     RECT
    FRestore    BOOL
    FIncUpdate  BOOL
    RgbReserved [32]byte
}

type SERVICE_STATUS struct {
    DwServiceType             DWORD
    DwCurrentState            DWORD
    DwControlsAccepted        DWORD
    DwWin32ExitCode           DWORD
    DwServiceSpecificExitCode DWORD
    DwCheckPoint              DWORD
    DwWaitHint                DWORD
}
<commit_msg>Thanks dreamkxd! Update the data type inconsistent issue of COLORREF and HRESULT.<commit_after>// Copyright 2010 The W32 Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package w32

import (
    "unsafe"
)

type (
    BOOL         int
    ATOM         uint16
    DWORD        uint32
    HANDLE       uintptr
    HINSTANCE    HANDLE
    HACCEL       HANDLE
    HCURSOR      HANDLE
    HDWP         HANDLE
    HICON        HANDLE
    HMENU        HANDLE
    HWND         HANDLE
    HBRUSH       HANDLE
    HRESULT      int32
    HFONT        HANDLE
    HDC          HANDLE
    HGDIOBJ      HANDLE
    HDROP        HANDLE
    HENHMETAFILE HANDLE
    HBITMAP      HANDLE
    HPEN         HANDLE
    HKEY         HANDLE
    HIMAGELIST   HANDLE
    HGLOBAL      HANDLE
    HMODULE      HANDLE
    HRSRC        HANDLE
    COLORREF     DWORD
)

type POINT struct {
    X, Y int
}

type RECT struct {
    Left, Top, Right, Bottom int
}

type WNDCLASSEX struct {
    Size       uint
    Style      uint
    WndProc    uintptr
    ClsExtra   int
    WndExtra   int
    Instance   HINSTANCE
    Icon       HICON
    Cursor     HCURSOR
    Background HBRUSH
    MenuName   *uint16
    ClassName  *uint16
    IconSm     HICON
}

type MSG struct {
    Hwnd    HWND
    Message uint
    WParam  uintptr
    LParam  uintptr
    Time    uint
    Pt      POINT
}

type LOGFONT struct {
    Height         int
    Width          int
    Escapement     int
    Orientation    int
    Weight         int
    Italic         byte
    Underline      byte
    StrikeOut      byte
    CharSet        byte
    OutPrecision   byte
    ClipPrecision  byte
    Quality        byte
    PitchAndFamily byte
    FaceName       [LF_FACESIZE]uint16
}

type OPENFILENAME struct {
    StructSize      uint
    Owner           HWND
    Instance        HINSTANCE
    Filter          *uint16
    CustomFilter    *uint16
    MaxCustomFilter uint
    FilterIndex     uint
    File            *uint16
    MaxFile         uint
    FileTitle       *uint16
    MaxFileTitle    uint
    InitialDir      *uint16
    Title           *uint16
    Flags           uint
    FileOffset      uint16
    FileExtension   uint16
    DefExt          *uint16
    CustData        uintptr
    FnHook          uintptr
    TemplateName    *uint16
    PvReserved      unsafe.Pointer
    DwReserved      uint
    FlagsEx         uint
}

type BROWSEINFO struct {
    Owner        HWND
    Root         *uint16
    DisplayName  *uint16
    Title        *uint16
    Flags        uint
    CallbackFunc uintptr
    LParam       uintptr
    Image        int
}

type GUID struct {
    Data1 uint
    Data2 uint16
    Data3 uint16
    Data4 [8]byte
}

type VARIANT struct {
    VT         uint16 //  2
    WReserved1 uint16 //  4
    WReserved2 uint16 //  6
    WReserved3 uint16 //  8
    Val        int64  // 16
}

type DISPPARAMS struct {
    Rgvarg            uintptr
    RgdispidNamedArgs uintptr
    CArgs             uint
    CNamedArgs        uint
}

type EXCEPINFO struct {
    WCode             uint16
    WReserved         uint16
    BstrSource        *uint16
    BstrDescription   *uint16
    BstrHelpFile      *uint16
    DwHelpContext     uint
    PvReserved        uintptr
    PfnDeferredFillIn uintptr
    Scode             int
}

type LOGBRUSH struct {
    LbStyle uint
    LbColor COLORREF
    LbHatch uintptr
}

type DEVMODE struct {
    DmDeviceName       [CCHDEVICENAME]uint16
    DmSpecVersion      uint16
    DmDriverVersion    uint16
    DmSize             uint16
    DmDriverExtra      uint16
    DmFields           uint
    DmOrientation      int16
    DmPaperSize        int16
    DmPaperLength      int16
    DmPaperWidth       int16
    DmScale            int16
    DmCopies           int16
    DmDefaultSource    int16
    DmPrintQuality     int16
    DmColor            int16
    DmDuplex           int16
    DmYResolution      int16
    DmTTOption         int16
    DmCollate          int16
    DmFormName         [CCHFORMNAME]uint16
    DmLogPixels        uint16
    DmBitsPerPel       uint
    DmPelsWidth        uint
    DmPelsHeight       uint
    DmDisplayFlags     uint
    DmDisplayFrequency uint
    DmICMMethod        uint
    DmICMIntent        uint
    DmMediaType        uint
    DmDitherType       uint
    DmReserved1        uint
    DmReserved2        uint
    DmPanningWidth     uint
    DmPanningHeight    uint
}

type BITMAPINFOHEADER struct {
    BiSize          uint
    BiWidth         int
    BiHeight        int
    BiPlanes        uint16
    BiBitCount      uint16
    BiCompression   uint
    BiSizeImage     uint
    BiXPelsPerMeter int
    BiYPelsPerMeter int
    BiClrUsed       uint
    BiClrImportant  uint
}

type RGBQUAD struct {
    RgbBlue     byte
    RgbGreen    byte
    RgbRed      byte
    RgbReserved byte
}

type BITMAPINFO struct {
    BmiHeader BITMAPINFOHEADER
    BmiColors *RGBQUAD
}

type BITMAP struct {
    BmType       int32
    BmWidth      int32
    BmHeight     int32
    BmWidthBytes int32
    BmPlanes     uint16
    BmBitsPixel  uint16
    BmBits       unsafe.Pointer
}

type DIBSECTION struct {
    DsBm        BITMAP
    DsBmih      BITMAPINFOHEADER
    DsBitfields [3]uint32
    DshSection  HANDLE
    DsOffset    uint32
}

type ENHMETAHEADER struct {
    IType          uint
    NSize          uint
    RclBounds      RECT
    RclFrame       RECT
    DSignature     uint
    NVersion       uint
    NBytes         uint
    NRecords       uint
    NHandles       uint16
    SReserved      uint16
    NDescription   uint
    OffDescription uint
    NPalEntries    uint
    SzlDevice      SIZE
    SzlMillimeters SIZE
    CbPixelFormat  uint
    OffPixelFormat uint
    BOpenGL        uint
    SzlMicrometers SIZE
}

type SIZE struct {
    CX, CY int
}

type TEXTMETRIC struct {
    TmHeight           int
    TmAscent           int
    TmDescent          int
    TmInternalLeading  int
    TmExternalLeading  int
    TmAveCharWidth     int
    TmMaxCharWidth     int
    TmWeight           int
    TmOverhang         int
    TmDigitizedAspectX int
    TmDigitizedAspectY int
    TmFirstChar        uint16
    TmLastChar         uint16
    TmDefaultChar      uint16
    TmBreakChar        uint16
    TmItalic           byte
    TmUnderlined       byte
    TmStruckOut        byte
    TmPitchAndFamily   byte
    TmCharSet          byte
}

type DOCINFO struct {
    CbSize       int
    LpszDocName  *uint16
    LpszOutput   *uint16
    LpszDatatype *uint16
    FwType       uint
}

type NMHDR struct {
    HwndFrom HWND
    IdFrom   uintptr
    Code     uint
}

type LVCOLUMN struct {
    Mask       uint
    Fmt        int
    Cx         int
    PszText    *uint16
    CchTextMax int
    ISubItem   int
    IImage     int
    IOrder     int
}

type LVITEM struct {
    Mask       uint
    IItem      int
    ISubItem   int
    State      uint
    StateMask  uint
    PszText    *uint16
    CchTextMax int
    IImage     int
    LParam     uintptr
    IIndent    int
    IGroupId   int
    CColumns   uint
    PuColumns  uint
}

type LVHITTESTINFO struct {
    Pt       POINT
    Flags    uint
    IItem    int
    ISubItem int
    IGroup   int
}

type NMITEMACTIVATE struct {
    Hdr       NMHDR
    IItem     int
    ISubItem  int
    UNewState uint
    UOldState uint
    UChanged  uint
    PtAction  POINT
    LParam    uintptr
    UKeyFlags uint
}

type NMLISTVIEW struct {
    Hdr       NMHDR
    IItem     int
    ISubItem  int
    UNewState uint
    UOldState uint
    UChanged  uint
    PtAction  POINT
    LParam    uintptr
}

type NMLVDISPINFO struct {
    Hdr  NMHDR
    Item LVITEM
}

type INITCOMMONCONTROLSEX struct {
    DwSize uint
    DwICC  uint
}

type TOOLINFO struct {
    CbSize     uint
    UFlags     uint
    Hwnd       HWND
    UId        uintptr
    Rect       RECT
    Hinst      HINSTANCE
    LpszText   *uint16
    LParam     uintptr
    LpReserved unsafe.Pointer
}

type TRACKMOUSEEVENT struct {
    CbSize      uint
    DwFlags     uint
    HwndTrack   HWND
    DwHoverTime uint
}

type GdiplusStartupInput struct {
    GdiplusVersion           uint32
    DebugEventCallback       uintptr
    SuppressBackgroundThread BOOL
    SuppressExternalCodecs   BOOL
}

type GdiplusStartupOutput struct {
    NotificationHook   uintptr
    NotificationUnhook uintptr
}

type PAINTSTRUCT struct {
    Hdc         HDC
    FErase      BOOL
    RcPaint     RECT
    FRestore    BOOL
    FIncUpdate  BOOL
    RgbReserved [32]byte
}

type SERVICE_STATUS struct {
    DwServiceType             DWORD
    DwCurrentState            DWORD
    DwControlsAccepted        DWORD
    DwWin32ExitCode           DWORD
    DwServiceSpecificExitCode DWORD
    DwCheckPoint              DWORD
    DwWaitHint                DWORD
}
<|endoftext|>


<commit_before>package ui

import (
	"encoding/json"
	"image"
	"image/draw"
	"image/gif"
	"image/png"
	"log"
	"math"
	"os"
	"strings"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/model"
)

type Image struct {
	pos    int
	frames []*image.RGBA
}

func (i *Image) GetNextFrame() *image.RGBA {
	i.pos++
	if i.pos >= len(i.frames) {
		i.pos = 0
	}
	return i.frames[i.pos]
}

// GetPositionFrame returns the frame corresponding to the position given 0....1
func (i *Image) GetPositionFrame(position float64) *image.RGBA {
	frameNumber := math.Min(float64(len(i.frames)-1), math.Floor(position*float64(len(i.frames))))
	return i.frames[int(frameNumber)]
}

func (i *Image) GetNumFrames() int {
	return len(i.frames)
}

func (i *Image) GetFrame(frame int) *image.RGBA {
	return i.frames[frame]
}

func loadImage(src string) *Image {
	srcLower := strings.ToLower(src)

	if strings.Contains(srcLower, ".gif") {
		return loadGif(src)
	} else if strings.Contains(srcLower, ".png") {
		return loadPng(src)
	} else {
		log.Fatalf("Unknown image format: %s", src)
	}
	return nil
}

func loadPng(src string) *Image {
	file, err := os.Open(src)

	if err != nil {
		log.Fatalf("Could not open png '%s' : %s", src, err)
	}

	img, err := png.Decode(file)
	if err != nil {
		log.Fatalf("PNG decoding failed on image '%s' : %s", src, err)
	}

	return &Image{
		frames: []*image.RGBA{toRGBA(img)},
	}
}

func loadGif(src string) *Image {
	file, err := os.Open(src)

	if err != nil {
		log.Fatalf("Could not open gif '%s' : %s", src, err)
	}

	img, err := gif.DecodeAll(file)
	if err != nil {
		log.Fatalf("Gif decoding failed on image '%s' : %s", src, err)
	}

	var frames = []*image.RGBA{}

	for _, frame := range img.Image {
		frames = append(frames, toRGBA(frame))
	}

	return &Image{
		frames: frames,
	}
}

func toRGBA(in image.Image) *image.RGBA {
	bounds := in.Bounds()
	out := image.NewRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
	draw.Draw(out, out.Bounds(), in, bounds.Min, draw.Over)
	return out
}

/*
type Thing struct {
	Name   string `json:"name"`
	ID     string `json:"id"`
	Device Device
}

type Device struct {
	Name     string `json:"name"`
	ID       string `json:"id"`
	IDType   string `json:"idType"`
	Guid     string `json:"guid"`
	Channels []Channel
}

type Channel struct {
	Protocol string `json:"protocol"`
	Name     string `json:"channel"`
	ID       string `json:"id"`
}*/

var conn *ninja.Connection
var tasks []*request
var thingModel *ninja.ServiceClient

type request struct {
	thingType string
	protocol  string
	cb        func([]*ninja.ServiceClient, error)
}

func runTasks(params *json.RawMessage, topicKeys map[string]string) bool {

	for _, task := range tasks {
		go task.cb(getChannelServices(task.thingType, task.protocol))
	}

	return true
}

func startSearchTasks(c *ninja.Connection) {
	conn = c
	thingModel = conn.GetServiceClient("$home/services/ThingModel")

	thingModel.OnEvent("created", runTasks)
	thingModel.OnEvent("updated", runTasks)
	thingModel.OnEvent("deleted", runTasks)
}

func getChannelServicesContinuous(thingType string, protocol string, cb func([]*ninja.ServiceClient, error)) {

	tasks = append(tasks, &request{thingType, protocol, cb})

	cb(getChannelServices(thingType, protocol))
}

func getChannelServices(thingType string, protocol string) ([]*ninja.ServiceClient, error) {

	//time.Sleep(time.Second * 3)

	var things []model.Thing

	err := thingModel.Call("fetchByType", []interface{}{thingType}, &things, time.Second*10)
	//err = client.Call("fetch", "c7ac05e0-9999-4d93-bfe3-a0b4bb5e7e78", &thing)

	if err != nil {
		log.Fatalf("Failed calling fetchByType method: %s", err)
	}

	//spew.Dump(things)

	var services []*ninja.ServiceClient

	for _, thing := range things {

		// Handle more than one channel with same protocol
		channelTopic := getChannelTopic(&thing, protocol)
		if channelTopic != "" {
			services = append(services, conn.GetServiceClient(channelTopic))
		}
	}
	return services, nil
}

/*func listenToEvents(topic string, conn *mqtt.MqttClient) {

	filter, err := mqtt.NewTopicFilter(topic+"/event/+", 0)
	if err != nil {
		log.Fatalf("Boom, no good", err)
	}

	receipt, err := conn.StartSubscription(func(client *mqtt.MqttClient, message mqtt.Message) {
		nameFind := nameRegex.FindAllStringSubmatch(string(message.Payload()), -1)
		rssiFind := rssiRegex.FindAllStringSubmatch(string(message.Payload()), -1)

		if nameFind == nil {
			// Not a sticknfind
		} else {
			name := nameFind[0][1]
			rssi := rssiFind[0][1]
			spew.Dump("name", name, "rssi", rssi)

			p.tag = name
			p.rssi = rssi
		}

	}, filter)

	if err != nil {
		log.Fatalf("Boom, no good", err)
	}

	<-receipt

}*/

func getChannelTopic(thing *model.Thing, protocol string) string {

	if thing.Device == nil || thing.Device.Channels == nil {
		return ""
	}

	for _, channel := range *thing.Device.Channels {
		if channel.Protocol == protocol {
			if thing.Device == nil {
				//spew.Dump("NO device on thing!", thing)
				return ""
			} else {
				return "$device/" + thing.Device.ID + "/channel/" + channel.ID
			}
		}
	}

	return ""
}
<commit_msg>On thing create/update/delete mark interface as dirty, update every 5 seconds<commit_after>package ui

import (
	"encoding/json"
	"image"
	"image/draw"
	"image/gif"
	"image/png"
	"log"
	"math"
	"os"
	"strings"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/model"
)

type Image struct {
	pos    int
	frames []*image.RGBA
}

func (i *Image) GetNextFrame() *image.RGBA {
	i.pos++
	if i.pos >= len(i.frames) {
		i.pos = 0
	}
	return i.frames[i.pos]
}

// GetPositionFrame returns the frame corresponding to the position given 0....1
func (i *Image) GetPositionFrame(position float64) *image.RGBA {
	frameNumber := math.Min(float64(len(i.frames)-1), math.Floor(position*float64(len(i.frames))))
	return i.frames[int(frameNumber)]
}

func (i *Image) GetNumFrames() int {
	return len(i.frames)
}

func (i *Image) GetFrame(frame int) *image.RGBA {
	return i.frames[frame]
}

func loadImage(src string) *Image {
	srcLower := strings.ToLower(src)

	if strings.Contains(srcLower, ".gif") {
		return loadGif(src)
	} else if strings.Contains(srcLower, ".png") {
		return loadPng(src)
	} else {
		log.Fatalf("Unknown image format: %s", src)
	}
	return nil
}

func loadPng(src string) *Image {
	file, err := os.Open(src)

	if err != nil {
		log.Fatalf("Could not open png '%s' : %s", src, err)
	}

	img, err := png.Decode(file)
	if err != nil {
		log.Fatalf("PNG decoding failed on image '%s' : %s", src, err)
	}

	return &Image{
		frames: []*image.RGBA{toRGBA(img)},
	}
}

func loadGif(src string) *Image {
	file, err := os.Open(src)

	if err != nil {
		log.Fatalf("Could not open gif '%s' : %s", src, err)
	}

	img, err := gif.DecodeAll(file)
	if err != nil {
		log.Fatalf("Gif decoding failed on image '%s' : %s", src, err)
	}

	var frames = []*image.RGBA{}

	for _, frame := range img.Image {
		frames = append(frames, toRGBA(frame))
	}

	return &Image{
		frames: frames,
	}
}

func toRGBA(in image.Image) *image.RGBA {
	bounds := in.Bounds()
	out := image.NewRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
	draw.Draw(out, out.Bounds(), in, bounds.Min, draw.Over)
	return out
}

/*
type Thing struct {
	Name   string `json:"name"`
	ID     string `json:"id"`
	Device Device
}

type Device struct {
	Name     string `json:"name"`
	ID       string `json:"id"`
	IDType   string `json:"idType"`
	Guid     string `json:"guid"`
	Channels []Channel
}

type Channel struct {
	Protocol string `json:"protocol"`
	Name     string `json:"channel"`
	ID       string `json:"id"`
}*/

var conn *ninja.Connection
var tasks []*request
var thingModel *ninja.ServiceClient

type request struct {
	thingType string
	protocol  string
	cb        func([]*ninja.ServiceClient, error)
}

var dirty = false

func runTasks() {

	for _, task := range tasks {
		go func(t *request) {
			t.cb(getChannelServices(t.thingType, t.protocol))
		}(task)
	}

}

func startSearchTasks(c *ninja.Connection) {
	conn = c
	thingModel = conn.GetServiceClient("$home/services/ThingModel")

	setDirty := func(params *json.RawMessage, topicKeys map[string]string) bool {
		log.Println("Devices added/removed/updated. Marking dirty.")
		dirty = true
		return true
	}

	thingModel.OnEvent("created", setDirty)
	thingModel.OnEvent("updated", setDirty)
	thingModel.OnEvent("deleted", setDirty)

	go func() {
		time.Sleep(time.Second * 20)
		for {
			time.Sleep(time.Second * 5)
			if dirty {
				runTasks()
				dirty = false
			}
		}
	}()
}

func getChannelServicesContinuous(thingType string, protocol string, cb func([]*ninja.ServiceClient, error)) {

	tasks = append(tasks, &request{thingType, protocol, cb})

	cb(getChannelServices(thingType, protocol))
}

func getChannelServices(thingType string, protocol string) ([]*ninja.ServiceClient, error) {

	//time.Sleep(time.Second * 3)

	var things []model.Thing

	err := thingModel.Call("fetchByType", []interface{}{thingType}, &things, time.Second*10)
	//err = client.Call("fetch", "c7ac05e0-9999-4d93-bfe3-a0b4bb5e7e78", &thing)

	if err != nil {
		log.Fatalf("Failed calling fetchByType method: %s", err)
	}

	//spew.Dump(things)

	var services []*ninja.ServiceClient

	for _, thing := range things {

		// Handle more than one channel with same protocol
		channelTopic := getChannelTopic(&thing, protocol)
		if channelTopic != "" {
			services = append(services, conn.GetServiceClient(channelTopic))
		}
	}
	return services, nil
}

/*func listenToEvents(topic string, conn *mqtt.MqttClient) {

	filter, err := mqtt.NewTopicFilter(topic+"/event/+", 0)
	if err != nil {
		log.Fatalf("Boom, no good", err)
	}

	receipt, err := conn.StartSubscription(func(client *mqtt.MqttClient, message mqtt.Message) {
		nameFind := nameRegex.FindAllStringSubmatch(string(message.Payload()), -1)
		rssiFind := rssiRegex.FindAllStringSubmatch(string(message.Payload()), -1)

		if nameFind == nil {
			// Not a sticknfind
		} else {
			name := nameFind[0][1]
			rssi := rssiFind[0][1]
			spew.Dump("name", name, "rssi", rssi)

			p.tag = name
			p.rssi = rssi
		}

	}, filter)

	if err != nil {
		log.Fatalf("Boom, no good", err)
	}

	<-receipt

}*/

func getChannelTopic(thing *model.Thing, protocol string) string {

	if thing.Device == nil || thing.Device.Channels == nil {
		return ""
	}

	for _, channel := range *thing.Device.Channels {
		if channel.Protocol == protocol {
			if thing.Device == nil {
				//spew.Dump("NO device on thing!", thing)
				return ""
			} else {
				return "$device/" + thing.Device.ID + "/channel/" + channel.ID
			}
		}
	}

	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	"github.com/mopsalarm/go-pr0gramm-tags/store"
	"strings"
	"time"
)

type tagInfo struct {
	Id     int    `db:"id"`
	ItemId int    `db:"item_id"`
	Tag    string `db:"tag"`
}

func queryTags(db *sqlx.DB, firstTagId, count int, consumer func(tagInfo)) error {
	var tagInfos []tagInfo
	err := db.Select(&tagInfos,
		"SELECT id, item_id, lower(tag) as tag FROM tags WHERE id >= $1 ORDER BY id ASC LIMIT $2",
		firstTagId, count)

	if err == nil {
		for _, tagInfo := range tagInfos {
			consumer(tagInfo)
		}
	}

	return err
}

type postInfo struct {
	Id            int    `db:"id"`
	Flags         int    `db:"flags"`
	Score         int    `db:"score"`
	CreatedEpoch  int    `db:"created"`
	Promoted      bool   `db:"promoted"`
	Username      string `db:"username"`
	UserMark      int    `db:"mark"`
	HasText       bool   `db:"has_text"`
	HasAudio      bool   `db:"audio"`
	Width         int    `db:"width"`
	Controversial bool   `db:"is_controversial"`
}

func queryItems(db *sqlx.DB, firstItemId, itemCount int, consumer func(postInfo)) error {
	var postInfos []postInfo

	err := db.Select(&postInfos, `
		SELECT
			items.id,
			items.flags,
			items.created,
			items.audio,
			items.width,
			items.mark,
			items.up - items.down as score,
			items.promoted != 0 as promoted,
			lower(items.username) AS username,
			COALESCE(texts.has_text, FALSE) AS has_text,
			up>60 AND down>60 AND least(up, down)::float/greatest(up, down)>=0.7 as is_controversial
		FROM
			items
			LEFT JOIN items_text texts ON (items.id = texts.item_id)
		WHERE items.id >= $1 OR to_timestamp(items.created) > CURRENT_TIMESTAMP - interval '1day'
		ORDER BY items.id ASC LIMIT $2`, firstItemId, itemCount)

	if err == nil {
		for _, postInfo := range postInfos {
			consumer(postInfo)
		}
	}

	return err
}

func sizeCategories(width int) []string {
	switch {
	case width > 3800:
		return []string{"2160p", "4k"}

	case width > 1900:
		return []string{"1080p", "hd"}

	case width > 1200:
		return []string{"720p", "hd"}

	case width > 600:
		return []string{"sd"}

	default:
		return []string{"kartoffel"}
	}
}

func userMarkToString(mark int) string {
	switch mark {
	case 6:
		return "ftb"
	case 1:
		return "newfag"
	default:
		return ""
	}
}

func FetchUpdates(db *sqlx.DB, state store.StoreState) (store.IterStore, store.StoreState, bool) {
	builder := store.NewStoreBuilder(HashWord)

	itemCount := 10000
	{
		err := queryItems(db, state.LastItemId, itemCount, func(postInfo postInfo) {
			itemId := int32(-postInfo.Id)

			// Prefixes currently in-use:
			//  d: date
			//  f: flags
			//  s: score
			//  u: user
			//  q: quality
			//  m: mark (ftb, newfag)

			builder.Push("u:"+CleanString(postInfo.Username), itemId)

			switch {
			case postInfo.Flags&1 != 0:
				builder.Push("f:sfw", itemId)
			case postInfo.Flags&2 != 0:
				builder.Push("f:nsfw", itemId)
			case postInfo.Flags&4 != 0:
				builder.Push("f:nsfl", itemId)
			case postInfo.Flags&8 != 0:
				builder.Push("f:nsfp", itemId)
			}

			if postInfo.Promoted {
				builder.Push("f:top", itemId)
			}

			if postInfo.HasText {
				builder.Push("f:text", itemId)
			}

			if postInfo.HasAudio {
				builder.Push("f:sound", itemId)
			}

			if postInfo.Controversial {
				builder.Push("f:controversial", itemId)
			}

			// mark content of ftp and newfags special.
			if mark := userMarkToString(postInfo.UserMark); mark != "" {
				builder.Push("m:"+mark, itemId)
			}

			// add quality-tag
			for _, sizeCategory := range sizeCategories(postInfo.Width) {
				builder.Push("q:"+sizeCategory, itemId)
			}

			// date tags
			created := time.Unix(int64(postInfo.CreatedEpoch), 0)
			builder.Push(fmt.Sprintf("d:%04d", created.Year()), itemId)
			builder.Push(fmt.Sprintf("d:%04d:%02d", created.Year(), created.Month()), itemId)

			// sort posts into bins (size 100) by score.
			// a post with score 1125 will be put into bins 100, 200,... and 1100
			for bin := 1; bin <= postInfo.Score/100; bin++ {
				label := fmt.Sprintf("s:%d", (100 * bin))
				builder.Push(label, itemId)
			}

			// add a label for the real shitty content.
			if postInfo.Score < -300 {
				builder.Push("s:shit", itemId)
			}

			itemCount -= 1
			state.LastItemId = postInfo.Id
		})

		if err != nil {
			log.WithError(err).Warn("Could not fetch the list of post items")
			metricsUpdaterError.Inc(1)
		}
	}

	tagCount := 50000
	{
		err := queryTags(db, state.LastTagId, tagCount, func(info tagInfo) {
			itemId := int32(-info.ItemId)
			for _, word := range ExtractWords(info.Tag) {
				builder.Push(word, itemId)
			}

			if strings.ToLower(info.Tag) == "repost" {
				builder.Push("f:repost", itemId)
			}

			tagCount -= 1
			state.LastTagId = info.Id
		})

		if err != nil {
			log.WithError(err).Warn("Error while streaming from postgres")
			metricsUpdaterError.Inc(1)
		}
	}

	expectMore := tagCount == 0 || itemCount == 0
	return builder.Build(), state, expectMore
}
<commit_msg>Read up to 7 days of posts.<commit_after>package main

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	"github.com/mopsalarm/go-pr0gramm-tags/store"
	"strings"
	"time"
)

type tagInfo struct {
	Id     int    `db:"id"`
	ItemId int    `db:"item_id"`
	Tag    string `db:"tag"`
}

func queryTags(db *sqlx.DB, firstTagId, count int, consumer func(tagInfo)) error {
	var tagInfos []tagInfo
	err := db.Select(&tagInfos,
		"SELECT id, item_id, lower(tag) as tag FROM tags WHERE id >= $1 ORDER BY id ASC LIMIT $2",
		firstTagId, count)

	if err == nil {
		for _, tagInfo := range tagInfos {
			consumer(tagInfo)
		}
	}

	return err
}

type postInfo struct {
	Id            int    `db:"id"`
	Flags         int    `db:"flags"`
	Score         int    `db:"score"`
	CreatedEpoch  int    `db:"created"`
	Promoted      bool   `db:"promoted"`
	Username      string `db:"username"`
	UserMark      int    `db:"mark"`
	HasText       bool   `db:"has_text"`
	HasAudio      bool   `db:"audio"`
	Width         int    `db:"width"`
	Controversial bool   `db:"is_controversial"`
}

func queryItems(db *sqlx.DB, firstItemId, days, itemCount int, consumer func(postInfo)) error {
	var postInfos []postInfo

	err := db.Select(&postInfos, fmt.Sprintf(`
		SELECT
			items.id,
			items.flags,
			items.created,
			items.audio,
			items.width,
			items.mark,
			items.up - items.down as score,
			items.promoted != 0 as promoted,
			lower(items.username) AS username,
			COALESCE(texts.has_text, FALSE) AS has_text,
			up>60 AND down>60 AND least(up, down)::float/greatest(up, down)>=0.7 as is_controversial
		FROM
			items
			LEFT JOIN items_text texts ON (items.id = texts.item_id)
		WHERE items.id >= $1 OR to_timestamp(items.created) > CURRENT_TIMESTAMP - interval '%d days'
		ORDER BY items.id ASC LIMIT $2`, days), firstItemId, itemCount)

	if err == nil {
		for _, postInfo := range postInfos {
			consumer(postInfo)
		}
	}

	return err
}

func sizeCategories(width int) []string {
	switch {
	case width > 3800:
		return []string{"2160p", "4k"}

	case width > 1900:
		return []string{"1080p", "hd"}

	case width > 1200:
		return []string{"720p", "hd"}

	case width > 600:
		return []string{"sd"}

	default:
		return []string{"kartoffel"}
	}
}

func userMarkToString(mark int) string {
	switch mark {
	case 6:
		return "ftb"
	case 1:
		return "newfag"
	default:
		return ""
	}
}

func FetchUpdates(db *sqlx.DB, state store.StoreState) (store.IterStore, store.StoreState, bool) {
	builder := store.NewStoreBuilder(HashWord)

	itemCount := 15000
	{
		days := 7
		err := queryItems(db, state.LastItemId, days, itemCount, func(postInfo postInfo) {
			itemId := int32(-postInfo.Id)

			// Prefixes currently in-use:
			//  d: date
			//  f: flags
			//  s: score
			//  u: user
			//  q: quality
			//  m: mark (ftb, newfag)

			builder.Push("u:"+CleanString(postInfo.Username), itemId)

			switch {
			case postInfo.Flags&1 != 0:
				builder.Push("f:sfw", itemId)
			case postInfo.Flags&2 != 0:
				builder.Push("f:nsfw", itemId)
			case postInfo.Flags&4 != 0:
				builder.Push("f:nsfl", itemId)
			case postInfo.Flags&8 != 0:
				builder.Push("f:nsfp", itemId)
			}

			if postInfo.Promoted {
				builder.Push("f:top", itemId)
			}

			if postInfo.HasText {
				builder.Push("f:text", itemId)
			}

			if postInfo.HasAudio {
				builder.Push("f:sound", itemId)
			}

			if postInfo.Controversial {
				builder.Push("f:controversial", itemId)
			}

			// mark content of ftp and newfags special.
			if mark := userMarkToString(postInfo.UserMark); mark != "" {
				builder.Push("m:"+mark, itemId)
			}

			// add quality-tag
			for _, sizeCategory := range sizeCategories(postInfo.Width) {
				builder.Push("q:"+sizeCategory, itemId)
			}

			// date tags
			created := time.Unix(int64(postInfo.CreatedEpoch), 0)
			builder.Push(fmt.Sprintf("d:%04d", created.Year()), itemId)
			builder.Push(fmt.Sprintf("d:%04d:%02d", created.Year(), created.Month()), itemId)

			// sort posts into bins (size 100) by score.
			// a post with score 1125 will be put into bins 100, 200,... and 1100
			for bin := 1; bin <= postInfo.Score/100; bin++ {
				label := fmt.Sprintf("s:%d", (100 * bin))
				builder.Push(label, itemId)
			}

			// add a label for the real shitty content.
			if postInfo.Score < -300 {
				builder.Push("s:shit", itemId)
			}

			itemCount -= 1
			state.LastItemId = postInfo.Id
		})

		if err != nil {
			log.WithError(err).Warn("Could not fetch the list of post items")
			metricsUpdaterError.Inc(1)
		}
	}

	tagCount := 50000
	{
		err := queryTags(db, state.LastTagId, tagCount, func(info tagInfo) {
			itemId := int32(-info.ItemId)
			for _, word := range ExtractWords(info.Tag) {
				builder.Push(word, itemId)
			}

			if strings.ToLower(info.Tag) == "repost" {
				builder.Push("f:repost", itemId)
			}

			tagCount -= 1
			state.LastTagId = info.Id
		})

		if err != nil {
			log.WithError(err).Warn("Error while streaming from postgres")
			metricsUpdaterError.Inc(1)
		}
	}

	expectMore := tagCount == 0 || itemCount == 0
	return builder.Build(), state, expectMore
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc.  All rights reserved.
// Copyright 2016 the gousb Authors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package usb

/*
#include <libusb-1.0/libusb.h>

int submit(struct libusb_transfer *xfer);
void print_xfer(struct libusb_transfer *xfer);
int extract_data(struct libusb_transfer *xfer, void *data, int max, unsigned char *status);
*/
import "C"

import (
	"fmt"
	"log"
	"time"
	"unsafe"
)

//export iso_callback
func iso_callback(cptr unsafe.Pointer) {
	ch := *(*chan struct{})(cptr)
	close(ch)
}

func (end *endpoint) allocTransfer() *Transfer {
	const (
		iso_packets = 8 // 128 // 242
	)

	xfer := C.libusb_alloc_transfer(C.int(iso_packets))
	if xfer == nil {
		log.Printf("usb: transfer allocation failed?!")
		return nil
	}

	buf := make([]byte, iso_packets*end.EndpointInfo.MaxIsoPacket)
	done := make(chan struct{}, 1)

	xfer.dev_handle = end.Device.handle
	xfer.endpoint = C.uchar(end.Address)
	xfer._type = C.LIBUSB_TRANSFER_TYPE_ISOCHRONOUS

	xfer.buffer = (*C.uchar)((unsafe.Pointer)(&buf[0]))
	xfer.length = C.int(len(buf))
	xfer.num_iso_packets = iso_packets

	C.libusb_set_iso_packet_lengths(xfer, C.uint(end.EndpointInfo.MaxIsoPacket))
	/*
		pkts := *(*[]C.struct_libusb_packet_descriptor)(unsafe.Pointer(&reflect.SliceHeader{
			Data: uintptr(unsafe.Pointer(&xfer.iso_packet_desc)),
			Len:  iso_packets,
			Cap:  iso_packets,
		}))
	*/

	t := &Transfer{
		xfer: xfer,
		done: done,
		buf:  buf,
	}
	xfer.user_data = (unsafe.Pointer)(&t.done)

	return t
}

type Transfer struct {
	xfer *C.struct_libusb_transfer
	pkts []*C.struct_libusb_packet_descriptor
	done chan struct{}
	buf  []byte
}

func (t *Transfer) Submit(timeout time.Duration) error {
	//log.Printf("iso: submitting %#v", t.xfer)
	t.xfer.timeout = C.uint(timeout / time.Millisecond)
	if errno := C.submit(t.xfer); errno < 0 {
		return usbError(errno)
	}
	return nil
}

func (t *Transfer) Wait(b []byte) (n int, err error) {
	select {
	case <-time.After(10 * time.Second):
		return 0, fmt.Errorf("wait timed out after 10s")
	case <-t.done:
	}
	// Non-iso transfers:
	//n = int(t.xfer.actual_length)
	//copy(b, ((*[1 << 16]byte)(unsafe.Pointer(t.xfer.buffer)))[:n])

	//C.print_xfer(t.xfer)
	/*
		buf, offset := ((*[1 << 16]byte)(unsafe.Pointer(t.xfer.buffer))), 0
		for i, pkt := range *t.pkts {
			log.Printf("Type is %T", t.pkts)
			n += copy(b[n:], buf[offset:][:pkt.actual_length])
			offset += pkt.Length
			if pkt.status != 0 && err == nil {
				err = error(TransferStatus(pkt.status))
			}
		}
	*/
	var status uint8
	n = int(C.extract_data(t.xfer, unsafe.Pointer(&b[0]), C.int(len(b)), (*C.uchar)(unsafe.Pointer(&status))))
	if status != 0 {
		err = TransferStatus(status)
	}
	return n, err
}

func (t *Transfer) Close() error {
	C.libusb_free_transfer(t.xfer)
	return nil
}

func isochronous_xfer(e *endpoint, buf []byte, timeout time.Duration) (int, error) {
	t := e.allocTransfer()
	defer t.Close()

	if err := t.Submit(timeout); err != nil {
		log.Printf("iso: xfer failed to submit: %s", err)
		return 0, err
	}

	n, err := t.Wait(buf)
	if err != nil {
		log.Printf("iso: xfer failed: %s", err)
		return 0, err
	}
	return n, err
}
<commit_msg>Pass the max buffer size down to the alloc_transfer. Use the number of iso packets matching the buffer size. This guarantees that the transfer size is smaller or equal to the buffer. Device will transfer less data if iso transfer response does not utilize the maximum available number of iso packets per microframe or doesn't fill the packets entirely.<commit_after>// Copyright 2013 Google Inc.  All rights reserved.
// Copyright 2016 the gousb Authors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package usb

/*
#include <libusb-1.0/libusb.h>

int submit(struct libusb_transfer *xfer);
void print_xfer(struct libusb_transfer *xfer);
int extract_data(struct libusb_transfer *xfer, void *data, int max, unsigned char *status);
*/
import "C"

import (
	"fmt"
	"log"
	"time"
	"unsafe"
)

//export iso_callback
func iso_callback(cptr unsafe.Pointer) {
	ch := *(*chan struct{})(cptr)
	close(ch)
}

func (end *endpoint) allocTransfer(maxLen int) *Transfer {
	isoPacketSize := end.EndpointInfo.MaxIsoPacket
	// the larget the input buffer, the more packets we use in a single
	// transfer.
	numIsoPackets := maxLen / int(isoPacketSize)
	if numIsoPackets*int(isoPacketSize) < maxLen {
		numIsoPackets++
	}
	// arbitrary limit
	if numIsoPackets > 200 {
		numIsoPackets = 200
	}
	xfer := C.libusb_alloc_transfer(C.int(numIsoPackets))
	if xfer == nil {
		log.Printf("usb: transfer allocation failed?!")
		return nil
	}

	buf := make([]byte, numIsoPackets*int(end.EndpointInfo.MaxIsoPacket))
	done := make(chan struct{}, 1)

	xfer.dev_handle = end.Device.handle
	xfer.endpoint = C.uchar(end.Address)
	xfer._type = C.LIBUSB_TRANSFER_TYPE_ISOCHRONOUS

	xfer.buffer = (*C.uchar)((unsafe.Pointer)(&buf[0]))
	xfer.length = C.int(len(buf))
	xfer.num_iso_packets = C.int(numIsoPackets)

	C.libusb_set_iso_packet_lengths(xfer, C.uint(end.EndpointInfo.MaxIsoPacket))
	/*
		pkts := *(*[]C.struct_libusb_packet_descriptor)(unsafe.Pointer(&reflect.SliceHeader{
			Data: uintptr(unsafe.Pointer(&xfer.iso_packet_desc)),
			Len:  iso_packets,
			Cap:  iso_packets,
		}))
	*/

	t := &Transfer{
		xfer: xfer,
		done: done,
		buf:  buf,
	}
	xfer.user_data = (unsafe.Pointer)(&t.done)

	return t
}

type Transfer struct {
	xfer *C.struct_libusb_transfer
	pkts []*C.struct_libusb_packet_descriptor
	done chan struct{}
	buf  []byte
}

func (t *Transfer) Submit(timeout time.Duration) error {
	//log.Printf("iso: submitting %#v", t.xfer)
	t.xfer.timeout = C.uint(timeout / time.Millisecond)
	if errno := C.submit(t.xfer); errno < 0 {
		return usbError(errno)
	}
	return nil
}

func (t *Transfer) Wait(b []byte) (n int, err error) {
	select {
	case <-time.After(10 * time.Second):
		return 0, fmt.Errorf("wait timed out after 10s")
	case <-t.done:
	}
	// Non-iso transfers:
	//n = int(t.xfer.actual_length)
	//copy(b, ((*[1 << 16]byte)(unsafe.Pointer(t.xfer.buffer)))[:n])

	//C.print_xfer(t.xfer)
	/*
		buf, offset := ((*[1 << 16]byte)(unsafe.Pointer(t.xfer.buffer))), 0
		for i, pkt := range *t.pkts {
			log.Printf("Type is %T", t.pkts)
			n += copy(b[n:], buf[offset:][:pkt.actual_length])
			offset += pkt.Length
			if pkt.status != 0 && err == nil {
				err = error(TransferStatus(pkt.status))
			}
		}
	*/
	var status uint8
	n = int(C.extract_data(t.xfer, unsafe.Pointer(&b[0]), C.int(len(b)), (*C.uchar)(unsafe.Pointer(&status))))
	if status != 0 {
		err = TransferStatus(status)
	}
	return n, err
}

func (t *Transfer) Close() error {
	C.libusb_free_transfer(t.xfer)
	return nil
}

func isochronous_xfer(e *endpoint, buf []byte, timeout time.Duration) (int, error) {
	t := e.allocTransfer(len(buf))
	defer t.Close()

	if err := t.Submit(timeout); err != nil {
		log.Printf("iso: xfer failed to submit: %s", err)
		return 0, err
	}

	n, err := t.Wait(buf)
	if err != nil {
		log.Printf("iso: xfer failed: %s", err)
		return 0, err
	}
	return n, err
}
<|endoftext|>