<commit_before>package main

import (
	"flag"
	"os"
	"strings"

	"torpedobot/multibot"
)

var (
	slack               = flag.String("slack", "", "Comma separated list of Slack legacy tokens")
	telegram            = flag.String("telegram", "", "Comma separated list of Telegram bot keys")
	jabber              = flag.String("jabber", "", "Comma separated list of jabber creds, user@host.com:password,")
	skype               = flag.String("skype", "", "Comma separated list of dev.botframework.com creds, app_id:app_password,")
	kik               = flag.String("kik", "", "Comma separated list of Kik creds, username:api_key,")
	skype_incoming_addr = flag.String("skype_incoming_addr", "0.0.0.0:3978", "Listen on this address for incoming Skype messages")
	facebook = flag.String("facebook", "", "Comma separated list of Facebook creds, page_token1:verify_token1,..")
	facebook_incoming_addr = flag.String("facebook_incoming_addr", "0.0.0.0:3979", "Listen on this address for incoming Facebook messages")
	kik_incoming_addr = flag.String("kik_incoming_addr", "0.0.0.0:3980", "Listen on this address for incoming Kik messages")
	kik_webhook_url = flag.String("kik_webhook_url", "", "Webhook URL (external) for incoming Kik messages")
	handlers            = make(map[string]func(*multibot.TorpedoBotAPI, interface{}, string))
)

func GetStripEnv(envvar string) (result string) {
	result = os.Getenv(envvar)
	result = strings.TrimLeft(result, "'")
	result = strings.TrimRight(result, "'")
	return
}

func main() {
	flag.Parse()
	handlers["bashim"] = BashProcessMessage
	handlers["bashorg"] = BashOrgProcessMessage
	handlers["lastfm"] = LastFmProcessMessage
	handlers["qr"] = QREncoderProcessMessage
	handlers["tinyurl"] = TinyURLProcessMessage
	// in progress
	// handlers["pinterest"] = PinterestProcessMessage
	// Help handlers
	handlers["?"] = HelpProcessMessage
	handlers["h"] = HelpProcessMessage
	handlers["help"] = HelpProcessMessage
	// Multimedia
	handlers["wiki"] = WikiProcessMessage
	handlers["youtube"] = YoutubeProcessMessage
	// in progress
	// handlers["giphy"] = GiphyProcessMessage
	// Probably useless stuff
	handlers["b64e"] = CryptoProcessMessage
	handlers["b64d"] = CryptoProcessMessage
	handlers["md5"] = CryptoProcessMessage
	handlers["sha1"] = CryptoProcessMessage
	handlers["sha256"] = CryptoProcessMessage
	handlers["sha512"] = CryptoProcessMessage
	// File category (broken!)
	//handlers["getimg"] = GetSetImageProcessMessage
	//handlers["lsimg"] = GetSetImageProcessMessage
	//handlers["listimg"] = GetSetImageProcessMessage
	//handlers["setimg"] = GetSetImageProcessMessage
	//handlers["rmimg"] = GetSetImageProcessMessage
	// Games'n'stuff
	// needs better formatting + Facebook rework
	//handlers["steam"] = SteamProcessMessage

	bot := multibot.New(*facebook_incoming_addr, *skype_incoming_addr, *kik_incoming_addr, *kik_webhook_url)
	bot.RegisterHandlers(handlers)
	if *slack == "" {
		*slack = GetStripEnv("SLACK")
	}
	if *telegram == "" {
		*telegram = GetStripEnv("TELEGRAM")
	}
	if *jabber == "" {
		*jabber = GetStripEnv("JABBER")
	}
	if *skype == "" {
		*skype = GetStripEnv("SKYPE")
	}
	if *facebook == "" {
		*facebook = GetStripEnv("FACEBOOK")
	}
	if *kik == "" {
		*kik = GetStripEnv("KIK")
	}
	if *kik_webhook_url == "" {
		*kik_webhook_url = GetStripEnv("KIK_WEBHOOK_URL")
	}
	if *lastfm_key == "" {
		*lastfm_key = GetStripEnv("LASTFM_KEY")
	}
	if *lastfm_secret == "" {
		*lastfm_secret = GetStripEnv("LASTFM_SECRET")
	}
	if *pinterest_token == "" {
		*pinterest_token = GetStripEnv("PINTEREST")
	}
	if *google_webapp_key == "" {
		*google_webapp_key = GetStripEnv("GOOGLE_WEBAPP_KEY")
	}
	bot.RunBotsCSV(bot.RunSlackBot, *slack, "!")
	bot.RunBotsCSV(bot.RunTelegramBot, *telegram, "/")
	bot.RunBotsCSV(bot.RunJabberBot, *jabber, "!")
	bot.RunBotsCSV(bot.RunSkypeBot, *skype, "!")
	bot.RunBotsCSV(bot.RunFacebookBot, *facebook, "!")
	bot.RunBotsCSV(bot.RunKikBot, *kik, "!")
	bot.RunLoop()
}
<commit_msg>vars..<commit_after>package main

import (
	"flag"
	"os"
	"strings"

	"torpedobot/multibot"
)

var (
	slack               = flag.String("slack", "", "Comma separated list of Slack legacy tokens")
	telegram            = flag.String("telegram", "", "Comma separated list of Telegram bot keys")
	jabber              = flag.String("jabber", "", "Comma separated list of jabber creds, user@host.com:password,")
	skype               = flag.String("skype", "", "Comma separated list of dev.botframework.com creds, app_id:app_password,")
	kik               = flag.String("kik", "", "Comma separated list of Kik creds, username:api_key,")
	skype_incoming_addr = flag.String("skype_incoming_addr", "0.0.0.0:3978", "Listen on this address for incoming Skype messages")
	facebook = flag.String("facebook", "", "Comma separated list of Facebook creds, page_token1:verify_token1,..")
	facebook_incoming_addr = flag.String("facebook_incoming_addr", "0.0.0.0:3979", "Listen on this address for incoming Facebook messages")
	kik_incoming_addr = flag.String("kik_incoming_addr", "0.0.0.0:3980", "Listen on this address for incoming Kik messages")
	kik_webhook_url = flag.String("kik_webhook_url", "", "Webhook URL (external) for incoming Kik messages")
	handlers            = make(map[string]func(*multibot.TorpedoBotAPI, interface{}, string))
)

func GetStripEnv(envvar string) (result string) {
	result = os.Getenv(envvar)
	result = strings.TrimLeft(result, "'")
	result = strings.TrimRight(result, "'")
	return
}

func main() {
	flag.Parse()
	handlers["bashim"] = BashProcessMessage
	handlers["bashorg"] = BashOrgProcessMessage
	handlers["lastfm"] = LastFmProcessMessage
	handlers["qr"] = QREncoderProcessMessage
	handlers["tinyurl"] = TinyURLProcessMessage
	// in progress
	// handlers["pinterest"] = PinterestProcessMessage
	// Help handlers
	handlers["?"] = HelpProcessMessage
	handlers["h"] = HelpProcessMessage
	handlers["help"] = HelpProcessMessage
	// Multimedia
	handlers["wiki"] = WikiProcessMessage
	handlers["youtube"] = YoutubeProcessMessage
	// in progress
	// handlers["giphy"] = GiphyProcessMessage
	// Probably useless stuff
	handlers["b64e"] = CryptoProcessMessage
	handlers["b64d"] = CryptoProcessMessage
	handlers["md5"] = CryptoProcessMessage
	handlers["sha1"] = CryptoProcessMessage
	handlers["sha256"] = CryptoProcessMessage
	handlers["sha512"] = CryptoProcessMessage
	// File category (broken!)
	//handlers["getimg"] = GetSetImageProcessMessage
	//handlers["lsimg"] = GetSetImageProcessMessage
	//handlers["listimg"] = GetSetImageProcessMessage
	//handlers["setimg"] = GetSetImageProcessMessage
	//handlers["rmimg"] = GetSetImageProcessMessage
	// Games'n'stuff
	// needs better formatting + Facebook rework
	//handlers["steam"] = SteamProcessMessage

	if *slack == "" {
		*slack = GetStripEnv("SLACK")
	}
	if *telegram == "" {
		*telegram = GetStripEnv("TELEGRAM")
	}
	if *jabber == "" {
		*jabber = GetStripEnv("JABBER")
	}
	if *skype == "" {
		*skype = GetStripEnv("SKYPE")
	}
	if *facebook == "" {
		*facebook = GetStripEnv("FACEBOOK")
	}
	if *kik == "" {
		*kik = GetStripEnv("KIK")
	}
	if *kik_webhook_url == "" {
		*kik_webhook_url = GetStripEnv("KIK_WEBHOOK_URL")
	}
	if *lastfm_key == "" {
		*lastfm_key = GetStripEnv("LASTFM_KEY")
	}
	if *lastfm_secret == "" {
		*lastfm_secret = GetStripEnv("LASTFM_SECRET")
	}
	if *pinterest_token == "" {
		*pinterest_token = GetStripEnv("PINTEREST")
	}
	if *google_webapp_key == "" {
		*google_webapp_key = GetStripEnv("GOOGLE_WEBAPP_KEY")
	}

	bot := multibot.New(*facebook_incoming_addr, *skype_incoming_addr, *kik_incoming_addr, *kik_webhook_url)
	bot.RegisterHandlers(handlers)

	bot.RunBotsCSV(bot.RunSlackBot, *slack, "!")
	bot.RunBotsCSV(bot.RunTelegramBot, *telegram, "/")
	bot.RunBotsCSV(bot.RunJabberBot, *jabber, "!")
	bot.RunBotsCSV(bot.RunSkypeBot, *skype, "!")
	bot.RunBotsCSV(bot.RunFacebookBot, *facebook, "!")
	bot.RunBotsCSV(bot.RunKikBot, *kik, "!")
	bot.RunLoop()
}
<|endoftext|>


<commit_before>package listen

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
)

func NewFlag(flagName, defaultValue string, serverType string) *Addr {
	addr := &Addr{
		s: defaultValue,
	}
	flag.Var(addr, flagName, Usage(serverType))
	return addr
}

// Usage returns a descriptive usage message for a flag given the name
// of thing being addressed.
func Usage(name string) string {
	if name == "" {
		name = "Listen address"
	}
	if !strings.HasSuffix(name, " address") {
		name += " address"
	}
	return name + "; may be port, :port, ip:port, FD:<fd_num>, or ADDR:<name> to use named runsit ports"
}

// Addr is a flag variable.  Use like:
//
// var webPort listen.Addr
// flag.Var(&webPort, "web_addr", listen.Usage("Web server address"))
// flag.Parse()
// webListener, err := webPort.Listen()
type Addr struct {
	s   string
	ln  net.Listener
	err error
}

func (a *Addr) String() string {
	return a.s
}

// Set implements the flag.Value interface.
func (a *Addr) Set(v string) error {
	a.s = v

	// Try the requested port by runsit port name first.
	fd, ok, err := namedPort(v)
	if err != nil {
		return err
	}
	if ok {
		return a.listenOnFD(fd)
	}

	if strings.HasPrefix(v, "FD:") {
		fdStr := v[len("FD:"):]
		fd, err := strconv.ParseUint(fdStr, 10, 32)
		if err != nil {
			return fmt.Errorf("invalid file descriptor %q: %v", fdStr, err)
		}
		return a.listenOnFD(uintptr(fd))
	}

	ipPort := v
	if isPort(v) {
		ipPort = ":" + v
	}

	_, _, err = net.SplitHostPort(ipPort)
	if err != nil {
		return fmt.Errorf("invalid PORT or IP:PORT %q: %v", v, err)
	}
	a.ln, err = net.Listen("tcp", ipPort)
	return err
}

func isPort(s string) bool {
	_, err := strconv.ParseUint(s, 10, 16)
	return err == nil
}

func (a *Addr) listenOnFD(fd uintptr) (err error) {
	f := os.NewFile(fd, fmt.Sprintf("fd #%d from runsit parent", fd))
	a.ln, err = net.FileListener(f)
	return
}

func namedPort(name string) (fd uintptr, ok bool, err error) {
	s := os.Getenv("RUNSIT_PORTFD_" + name)
	if s == "" {
		return
	}
	u64, err := strconv.ParseUint(s, 10, 32)
	if err != nil {
		return
	}
	return uintptr(u64), true, nil
}

var _ flag.Value = (*Addr)(nil)

// Listen returns the address's TCP listener.
func (a *Addr) Listen() (net.Listener, error) {
	// Start the listener now, if there's a default
	// and nothing's called Set yet.
	if a.err == nil && a.ln == nil && a.s != "" {
		if err := a.Set(a.s); err != nil {
			return nil, err
		}
	}
	if a.err != nil {
		return nil, a.err
	}
	if a.ln != nil {
		return a.ln, nil
	}
	return nil, errors.New("listen: no error or listener")
}
<commit_msg>add Listen function<commit_after>package listen

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
)

// NewFlag returns a flag that implements the flag.Value interface.
func NewFlag(flagName, defaultValue string, serverType string) *Addr {
	addr := &Addr{
		s: defaultValue,
	}
	flag.Var(addr, flagName, Usage(serverType))
	return addr
}

// Listen is a replacement for net.Listen that also respects runsit
// listeners: port, :port, ip:port, FD:<fd_num>, ADDR:<name> or <name>
// named ports.
// Listeners are always TCP.
func Listen(addr string) (net.Listener, error) {
	a := &Addr{s: addr}
	return a.Listen()
}

// Usage returns a descriptive usage message for a flag given the name
// of thing being addressed.
func Usage(name string) string {
	if name == "" {
		name = "Listen address"
	}
	if !strings.HasSuffix(name, " address") {
		name += " address"
	}
	return name + "; may be port, :port, ip:port, FD:<fd_num>, or ADDR:<name> to use named runsit ports"
}

// Addr is a flag variable.  Use like:
//
// var webPort listen.Addr
// flag.Var(&webPort, "web_addr", listen.Usage("Web server address"))
// flag.Parse()
// webListener, err := webPort.Listen()
type Addr struct {
	s   string
	ln  net.Listener
	err error
}

func (a *Addr) String() string {
	return a.s
}

// Set implements the flag.Value interface.
func (a *Addr) Set(v string) error {
	a.s = v

	// Try the requested port by runsit port name first.
	fd, ok, err := namedPort(v)
	if err != nil {
		return err
	}
	if ok {
		return a.listenOnFD(fd)
	}

	if strings.HasPrefix(v, "FD:") {
		fdStr := v[len("FD:"):]
		fd, err := strconv.ParseUint(fdStr, 10, 32)
		if err != nil {
			return fmt.Errorf("invalid file descriptor %q: %v", fdStr, err)
		}
		return a.listenOnFD(uintptr(fd))
	}

	ipPort := v
	if isPort(v) {
		ipPort = ":" + v
	}

	_, _, err = net.SplitHostPort(ipPort)
	if err != nil {
		return fmt.Errorf("invalid PORT or IP:PORT %q: %v", v, err)
	}
	a.ln, err = net.Listen("tcp", ipPort)
	return err
}

func isPort(s string) bool {
	_, err := strconv.ParseUint(s, 10, 16)
	return err == nil
}

func (a *Addr) listenOnFD(fd uintptr) (err error) {
	f := os.NewFile(fd, fmt.Sprintf("fd #%d from runsit parent", fd))
	a.ln, err = net.FileListener(f)
	return
}

func namedPort(name string) (fd uintptr, ok bool, err error) {
	s := os.Getenv("RUNSIT_PORTFD_" + name)
	if s == "" {
		return
	}
	u64, err := strconv.ParseUint(s, 10, 32)
	if err != nil {
		return
	}
	return uintptr(u64), true, nil
}

var _ flag.Value = (*Addr)(nil)

// Listen returns the address's TCP listener.
func (a *Addr) Listen() (net.Listener, error) {
	// Start the listener now, if there's a default
	// and nothing's called Set yet.
	if a.err == nil && a.ln == nil && a.s != "" {
		if err := a.Set(a.s); err != nil {
			return nil, err
		}
	}
	if a.err != nil {
		return nil, a.err
	}
	if a.ln != nil {
		return a.ln, nil
	}
	return nil, errors.New("listen: no error or listener")
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
)

func TestHoverflyListener(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	proxyPort := "9777"

	dbClient.Cfg.ProxyPort = proxyPort
	// starting hoverfly
	proxy, _ := GetNewHoverfly(dbClient.Cfg, dbClient.Cache)
	StartHoverflyProxy(dbClient.Cfg, proxy)

	// checking whether it's running
	response, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	expect(t, err, nil)

	expect(t, response.StatusCode, 500)

	body, err := ioutil.ReadAll(response.Body)
	expect(t, err, nil)
	expect(t, strings.Contains(string(body), "is a proxy server"), true)
}

func TestStopHoverflyListener(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	proxyPort := "9778"

	dbClient.Cfg.ProxyPort = proxyPort
	// starting hoverfly
	proxy, _ := GetNewHoverfly(dbClient.Cfg, dbClient.Cache)
	StartHoverflyProxy(dbClient.Cfg, proxy)

	dbClient.Cfg.SL.Stop()
	dbClient.Cfg.ProxyControlWG.Wait()

	// checking whether it's stopped
	_, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	// should get error
	refute(t, err, nil)
}
<commit_msg>test restart hoverfly<commit_after>package hoverfly

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
)

func TestHoverflyListener(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	proxyPort := "9777"

	dbClient.Cfg.ProxyPort = proxyPort
	// starting hoverfly
	proxy, _ := GetNewHoverfly(dbClient.Cfg, dbClient.Cache)
	StartHoverflyProxy(dbClient.Cfg, proxy)

	// checking whether it's running
	response, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	expect(t, err, nil)

	expect(t, response.StatusCode, 500)

	body, err := ioutil.ReadAll(response.Body)
	expect(t, err, nil)
	expect(t, strings.Contains(string(body), "is a proxy server"), true)
}

func TestStopHoverflyListener(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	proxyPort := "9778"

	dbClient.Cfg.ProxyPort = proxyPort
	// starting hoverfly
	proxy, _ := GetNewHoverfly(dbClient.Cfg, dbClient.Cache)
	StartHoverflyProxy(dbClient.Cfg, proxy)

	dbClient.Cfg.SL.Stop()
	dbClient.Cfg.ProxyControlWG.Wait()

	// checking whether it's stopped
	_, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	// should get error
	refute(t, err, nil)
}

func TestRestartHoverflyListener(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	proxyPort := "9779"

	dbClient.Cfg.ProxyPort = proxyPort
	// starting hoverfly
	proxy, _ := GetNewHoverfly(dbClient.Cfg, dbClient.Cache)
	StartHoverflyProxy(dbClient.Cfg, proxy)

	// checking whether it's running
	response, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	expect(t, err, nil)

	expect(t, response.StatusCode, 500)

	// stopping proxy
	dbClient.Cfg.SL.Stop()
	dbClient.Cfg.ProxyControlWG.Wait()

	// starting again
	StartHoverflyProxy(dbClient.Cfg, proxy)

	newResponse, err := http.Get(fmt.Sprintf("http://localhost:%s/", proxyPort))
	expect(t, err, nil)
	expect(t, newResponse.StatusCode, 500)
}
<|endoftext|>


<commit_before>package lncfg

import (
	"context"
	"crypto/tls"
	"encoding/hex"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/btcsuite/btcd/btcec/v2"
	"github.com/lightningnetwork/lnd/lnwire"
	"github.com/lightningnetwork/lnd/tor"
)

var (
	loopBackAddrs = []string{"localhost", "127.0.0.1", "[::1]"}
)

// TCPResolver is a function signature that resolves an address on a given
// network.
type TCPResolver = func(network, addr string) (*net.TCPAddr, error)

// NormalizeAddresses returns a new slice with all the passed addresses
// normalized with the given default port and all duplicates removed.
func NormalizeAddresses(addrs []string, defaultPort string,
	tcpResolver TCPResolver) ([]net.Addr, error) {

	result := make([]net.Addr, 0, len(addrs))
	seen := map[string]struct{}{}

	for _, addr := range addrs {
		parsedAddr, err := ParseAddressString(
			addr, defaultPort, tcpResolver,
		)
		if err != nil {
			return nil, fmt.Errorf("parse address %s failed: %w",
				addr, err)
		}

		if _, ok := seen[parsedAddr.String()]; !ok {
			result = append(result, parsedAddr)
			seen[parsedAddr.String()] = struct{}{}
		}
	}

	return result, nil
}

// EnforceSafeAuthentication enforces "safe" authentication taking into account
// the interfaces that the RPC servers are listening on, and if macaroons and
// TLS is activated or not. To protect users from using dangerous config
// combinations, we'll prevent disabling authentication if the server is
// listening on a public interface.
func EnforceSafeAuthentication(addrs []net.Addr, macaroonsActive,
	tlsActive bool) error {

	// We'll now examine all addresses that this RPC server is listening
	// on. If it's a localhost address or a private address, we'll skip it,
	// otherwise, we'll return an error if macaroons are inactive.
	for _, addr := range addrs {
		if IsLoopback(addr.String()) || IsUnix(addr) || IsPrivate(addr) {
			continue
		}

		if !macaroonsActive {
			return fmt.Errorf("detected RPC server listening on "+
				"publicly reachable interface %v with "+
				"authentication disabled! Refusing to start "+
				"with --no-macaroons specified", addr)
		}

		if !tlsActive {
			return fmt.Errorf("detected RPC server listening on "+
				"publicly reachable interface %v with "+
				"encryption disabled! Refusing to start "+
				"with --no-rest-tls specified", addr)
		}
	}

	return nil
}

// parseNetwork parses the network type of the given address.
func parseNetwork(addr net.Addr) string {
	switch addr := addr.(type) {
	// TCP addresses resolved through net.ResolveTCPAddr give a default
	// network of "tcp", so we'll map back the correct network for the given
	// address. This ensures that we can listen on the correct interface
	// (IPv4 vs IPv6).
	case *net.TCPAddr:
		if addr.IP.To4() != nil {
			return "tcp4"
		}
		return "tcp6"

	default:
		return addr.Network()
	}
}

// ListenOnAddress creates a listener that listens on the given address.
func ListenOnAddress(addr net.Addr) (net.Listener, error) {
	return net.Listen(parseNetwork(addr), addr.String())
}

// TLSListenOnAddress creates a TLS listener that listens on the given address.
func TLSListenOnAddress(addr net.Addr,
	config *tls.Config) (net.Listener, error) {
	return tls.Listen(parseNetwork(addr), addr.String(), config)
}

// IsLoopback returns true if an address describes a loopback interface.
func IsLoopback(addr string) bool {
	for _, loopback := range loopBackAddrs {
		if strings.Contains(addr, loopback) {
			return true
		}
	}

	return false
}

// isIPv6Host returns true if the host is IPV6 and false otherwise.
func isIPv6Host(host string) bool {
	v6Addr := net.ParseIP(host)
	if v6Addr == nil {
		return false
	}

	// The documentation states that if the IP address is an IPv6 address,
	// then To4() will return nil.
	return v6Addr.To4() == nil
}

// isUnspecifiedHost returns true if the host IP is considered unspecified.
func isUnspecifiedHost(host string) bool {
	addr := net.ParseIP(host)
	if addr == nil {
		return false
	}

	return addr.IsUnspecified()
}

// IsUnix returns true if an address describes an Unix socket address.
func IsUnix(addr net.Addr) bool {
	return strings.HasPrefix(addr.Network(), "unix")
}

// IsPrivate returns true if the address is private. The definitions are,
//   https://en.wikipedia.org/wiki/Link-local_address
//   https://en.wikipedia.org/wiki/Multicast_address
//   Local IPv4 addresses, https://tools.ietf.org/html/rfc1918
//   Local IPv6 addresses, https://tools.ietf.org/html/rfc4193
func IsPrivate(addr net.Addr) bool {
	switch addr := addr.(type) {
	case *net.TCPAddr:
		// Check 169.254.0.0/16 and fe80::/10.
		if addr.IP.IsLinkLocalUnicast() {
			return true
		}

		// Check 224.0.0.0/4 and ff00::/8.
		if addr.IP.IsLinkLocalMulticast() {
			return true
		}

		// Check 10.0.0.0/8, 172.16.0.0/12 and 192.168.0.0/16.
		if ip4 := addr.IP.To4(); ip4 != nil {
			return ip4[0] == 10 ||
				(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
				(ip4[0] == 192 && ip4[1] == 168)
		}

		// Check fc00::/7.
		return len(addr.IP) == net.IPv6len && addr.IP[0]&0xfe == 0xfc

	default:
		return false
	}
}

// ParseAddressString converts an address in string format to a net.Addr that is
// compatible with lnd. UDP is not supported because lnd needs reliable
// connections. We accept a custom function to resolve any TCP addresses so
// that caller is able control exactly how resolution is performed.
func ParseAddressString(strAddress string, defaultPort string,
	tcpResolver TCPResolver) (net.Addr, error) {

	var parsedNetwork, parsedAddr string

	// Addresses can either be in network://address:port format,
	// network:address:port, address:port, or just port. We want to support
	// all possible types.
	if strings.Contains(strAddress, "://") {
		parts := strings.Split(strAddress, "://")
		parsedNetwork, parsedAddr = parts[0], parts[1]
	} else if strings.Contains(strAddress, ":") {
		parts := strings.Split(strAddress, ":")
		parsedNetwork = parts[0]
		parsedAddr = strings.Join(parts[1:], ":")
	}

	// Only TCP and Unix socket addresses are valid. We can't use IP or
	// UDP only connections for anything we do in lnd.
	switch parsedNetwork {
	case "unix", "unixpacket":
		return net.ResolveUnixAddr(parsedNetwork, parsedAddr)

	case "tcp", "tcp4", "tcp6":
		return tcpResolver(
			parsedNetwork, verifyPort(parsedAddr, defaultPort),
		)

	case "ip", "ip4", "ip6", "udp", "udp4", "udp6", "unixgram":
		return nil, fmt.Errorf("only TCP or unix socket "+
			"addresses are supported: %s", parsedAddr)

	default:
		// We'll now possibly apply the default port, use the local
		// host short circuit, or parse out an all interfaces listen.
		addrWithPort := verifyPort(strAddress, defaultPort)
		rawHost, rawPort, _ := net.SplitHostPort(addrWithPort)

		// If we reach this point, then we'll check to see if we have
		// an onion addresses, if so, we can directly pass the raw
		// address and port to create the proper address.
		if tor.IsOnionHost(rawHost) {
			portNum, err := strconv.Atoi(rawPort)
			if err != nil {
				return nil, err
			}

			return &tor.OnionAddr{
				OnionService: rawHost,
				Port:         portNum,
			}, nil
		}

		// Otherwise, we'll attempt the resolve the host. The Tor
		// resolver is unable to resolve local addresses,
		// IPv6 addresses, or the all-interfaces address, so we'll use
		// the system resolver instead for those.
		if rawHost == "" || IsLoopback(rawHost) ||
			isIPv6Host(rawHost) || isUnspecifiedHost(rawHost) {

			return net.ResolveTCPAddr("tcp", addrWithPort)
		}

		// If we've reached this point, then it's possible that this
		// resolve returns an error if it isn't able to resolve the
		// host. For example, local entries in /etc/hosts will fail to
		// be resolved by Tor. In order to handle this case, we'll fall
		// back to the normal system resolver if we fail with an
		// identifiable error.
		addr, err := tcpResolver("tcp", addrWithPort)
		if err != nil {
			torErrStr := "tor host is unreachable"
			if strings.Contains(err.Error(), torErrStr) {
				return net.ResolveTCPAddr("tcp", addrWithPort)
			}

			return nil, err
		}

		return addr, nil
	}
}

// ParseLNAddressString converts a string of the form <pubkey>@<addr> into an
// lnwire.NetAddress. The <pubkey> must be presented in hex, and result in a
// 33-byte, compressed public key that lies on the secp256k1 curve. The <addr>
// may be any address supported by ParseAddressString. If no port is specified,
// the defaultPort will be used. Any tcp addresses that need resolving will be
// resolved using the custom TCPResolver.
func ParseLNAddressString(strAddress string, defaultPort string,
	tcpResolver TCPResolver) (*lnwire.NetAddress, error) {

	// Split the address string around the @ sign.
	parts := strings.Split(strAddress, "@")

	// The string is malformed if there are not exactly two parts.
	if len(parts) != 2 {
		return nil, fmt.Errorf("invalid lightning address %s: "+
			"must be of the form <pubkey-hex>@<addr>", strAddress)
	}

	// Now, take the first portion as the hex pubkey, and the latter as the
	// address string.
	parsedPubKey, parsedAddr := parts[0], parts[1]

	// Decode the hex pubkey to get the raw compressed pubkey bytes.
	pubKeyBytes, err := hex.DecodeString(parsedPubKey)
	if err != nil {
		return nil, fmt.Errorf("invalid lightning address pubkey: %v", err)
	}

	// The compressed pubkey should have a length of exactly 33 bytes.
	if len(pubKeyBytes) != 33 {
		return nil, fmt.Errorf("invalid lightning address pubkey: "+
			"length must be 33 bytes, found %d", len(pubKeyBytes))
	}

	// Parse the pubkey bytes to verify that it corresponds to valid public
	// key on the secp256k1 curve.
	pubKey, err := btcec.ParsePubKey(pubKeyBytes)
	if err != nil {
		return nil, fmt.Errorf("invalid lightning address pubkey: %v", err)
	}

	// Finally, parse the address string using our generic address parser.
	addr, err := ParseAddressString(parsedAddr, defaultPort, tcpResolver)
	if err != nil {
		return nil, fmt.Errorf("invalid lightning address address: %v", err)
	}

	return &lnwire.NetAddress{
		IdentityKey: pubKey,
		Address:     addr,
	}, nil
}

// verifyPort makes sure that an address string has both a host and a port. If
// there is no port found, the default port is appended. If the address is just
// a port, then we'll assume that the user is using the short cut to specify a
// localhost:port address.
func verifyPort(address string, defaultPort string) string {
	host, port, err := net.SplitHostPort(address)
	if err != nil {
		// If the address itself is just an integer, then we'll assume
		// that we're mapping this directly to a localhost:port pair.
		// This ensures we maintain the legacy behavior.
		if _, err := strconv.Atoi(address); err == nil {
			return net.JoinHostPort("localhost", address)
		}

		// Otherwise, we'll assume that the address just failed to
		// attach its own port, so we'll use the default port. In the
		// case of IPv6 addresses, if the host is already surrounded by
		// brackets, then we'll avoid using the JoinHostPort function,
		// since it will always add a pair of brackets.
		if strings.HasPrefix(address, "[") {
			return address + ":" + defaultPort
		}
		return net.JoinHostPort(address, defaultPort)
	}

	// In the case that both the host and port are empty, we'll use the
	// default port.
	if host == "" && port == "" {
		return ":" + defaultPort
	}

	return address
}

// ClientAddressDialer creates a gRPC dialer that can also dial unix socket
// addresses instead of just TCP addresses.
func ClientAddressDialer(defaultPort string) func(context.Context,
	string) (net.Conn, error) {

	return func(ctx context.Context, addr string) (net.Conn, error) {
		parsedAddr, err := ParseAddressString(
			addr, defaultPort, net.ResolveTCPAddr,
		)
		if err != nil {
			return nil, err
		}

		d := net.Dialer{}
		return d.DialContext(
			ctx, parsedAddr.Network(), parsedAddr.String(),
		)
	}
}
<commit_msg>lncfg: use net pkg IsLoopback method<commit_after>package lncfg

import (
	"context"
	"crypto/tls"
	"encoding/hex"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/btcsuite/btcd/btcec/v2"
	"github.com/lightningnetwork/lnd/lnwire"
	"github.com/lightningnetwork/lnd/tor"
)

// TCPResolver is a function signature that resolves an address on a given
// network.
type TCPResolver = func(network, addr string) (*net.TCPAddr, error)

// NormalizeAddresses returns a new slice with all the passed addresses
// normalized with the given default port and all duplicates removed.
func NormalizeAddresses(addrs []string, defaultPort string,
	tcpResolver TCPResolver) ([]net.Addr, error) {

	result := make([]net.Addr, 0, len(addrs))
	seen := map[string]struct{}{}

	for _, addr := range addrs {
		parsedAddr, err := ParseAddressString(
			addr, defaultPort, tcpResolver,
		)
		if err != nil {
			return nil, fmt.Errorf("parse address %s failed: %w",
				addr, err)
		}

		if _, ok := seen[parsedAddr.String()]; !ok {
			result = append(result, parsedAddr)
			seen[parsedAddr.String()] = struct{}{}
		}
	}

	return result, nil
}

// EnforceSafeAuthentication enforces "safe" authentication taking into account
// the interfaces that the RPC servers are listening on, and if macaroons and
// TLS is activated or not. To protect users from using dangerous config
// combinations, we'll prevent disabling authentication if the server is
// listening on a public interface.
func EnforceSafeAuthentication(addrs []net.Addr, macaroonsActive,
	tlsActive bool) error {

	// We'll now examine all addresses that this RPC server is listening
	// on. If it's a localhost address or a private address, we'll skip it,
	// otherwise, we'll return an error if macaroons are inactive.
	for _, addr := range addrs {
		if IsLoopback(addr.String()) || IsUnix(addr) || IsPrivate(addr) {
			continue
		}

		if !macaroonsActive {
			return fmt.Errorf("detected RPC server listening on "+
				"publicly reachable interface %v with "+
				"authentication disabled! Refusing to start "+
				"with --no-macaroons specified", addr)
		}

		if !tlsActive {
			return fmt.Errorf("detected RPC server listening on "+
				"publicly reachable interface %v with "+
				"encryption disabled! Refusing to start "+
				"with --no-rest-tls specified", addr)
		}
	}

	return nil
}

// parseNetwork parses the network type of the given address.
func parseNetwork(addr net.Addr) string {
	switch addr := addr.(type) {
	// TCP addresses resolved through net.ResolveTCPAddr give a default
	// network of "tcp", so we'll map back the correct network for the given
	// address. This ensures that we can listen on the correct interface
	// (IPv4 vs IPv6).
	case *net.TCPAddr:
		if addr.IP.To4() != nil {
			return "tcp4"
		}
		return "tcp6"

	default:
		return addr.Network()
	}
}

// ListenOnAddress creates a listener that listens on the given address.
func ListenOnAddress(addr net.Addr) (net.Listener, error) {
	return net.Listen(parseNetwork(addr), addr.String())
}

// TLSListenOnAddress creates a TLS listener that listens on the given address.
func TLSListenOnAddress(addr net.Addr,
	config *tls.Config) (net.Listener, error) {
	return tls.Listen(parseNetwork(addr), addr.String(), config)
}

// IsLoopback returns true if an address describes a loopback interface.
func IsLoopback(host string) bool {
	if strings.Contains(host, "localhost") {
		return true
	}

	rawHost, _, _ := net.SplitHostPort(host)
	addr := net.ParseIP(rawHost)
	if addr == nil {
		return false
	}

	return addr.IsLoopback()
}

// isIPv6Host returns true if the host is IPV6 and false otherwise.
func isIPv6Host(host string) bool {
	v6Addr := net.ParseIP(host)
	if v6Addr == nil {
		return false
	}

	// The documentation states that if the IP address is an IPv6 address,
	// then To4() will return nil.
	return v6Addr.To4() == nil
}

// isUnspecifiedHost returns true if the host IP is considered unspecified.
func isUnspecifiedHost(host string) bool {
	addr := net.ParseIP(host)
	if addr == nil {
		return false
	}

	return addr.IsUnspecified()
}

// IsUnix returns true if an address describes an Unix socket address.
func IsUnix(addr net.Addr) bool {
	return strings.HasPrefix(addr.Network(), "unix")
}

// IsPrivate returns true if the address is private. The definitions are,
//   https://en.wikipedia.org/wiki/Link-local_address
//   https://en.wikipedia.org/wiki/Multicast_address
//   Local IPv4 addresses, https://tools.ietf.org/html/rfc1918
//   Local IPv6 addresses, https://tools.ietf.org/html/rfc4193
func IsPrivate(addr net.Addr) bool {
	switch addr := addr.(type) {
	case *net.TCPAddr:
		// Check 169.254.0.0/16 and fe80::/10.
		if addr.IP.IsLinkLocalUnicast() {
			return true
		}

		// Check 224.0.0.0/4 and ff00::/8.
		if addr.IP.IsLinkLocalMulticast() {
			return true
		}

		// Check 10.0.0.0/8, 172.16.0.0/12 and 192.168.0.0/16.
		if ip4 := addr.IP.To4(); ip4 != nil {
			return ip4[0] == 10 ||
				(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
				(ip4[0] == 192 && ip4[1] == 168)
		}

		// Check fc00::/7.
		return len(addr.IP) == net.IPv6len && addr.IP[0]&0xfe == 0xfc

	default:
		return false
	}
}

// ParseAddressString converts an address in string format to a net.Addr that is
// compatible with lnd. UDP is not supported because lnd needs reliable
// connections. We accept a custom function to resolve any TCP addresses so
// that caller is able control exactly how resolution is performed.
func ParseAddressString(strAddress string, defaultPort string,
	tcpResolver TCPResolver) (net.Addr, error) {

	var parsedNetwork, parsedAddr string

	// Addresses can either be in network://address:port format,
	// network:address:port, address:port, or just port. We want to support
	// all possible types.
	if strings.Contains(strAddress, "://") {
		parts := strings.Split(strAddress, "://")
		parsedNetwork, parsedAddr = parts[0], parts[1]
	} else if strings.Contains(strAddress, ":") {
		parts := strings.Split(strAddress, ":")
		parsedNetwork = parts[0]
		parsedAddr = strings.Join(parts[1:], ":")
	}

	// Only TCP and Unix socket addresses are valid. We can't use IP or
	// UDP only connections for anything we do in lnd.
	switch parsedNetwork {
	case "unix", "unixpacket":
		return net.ResolveUnixAddr(parsedNetwork, parsedAddr)

	case "tcp", "tcp4", "tcp6":
		return tcpResolver(
			parsedNetwork, verifyPort(parsedAddr, defaultPort),
		)

	case "ip", "ip4", "ip6", "udp", "udp4", "udp6", "unixgram":
		return nil, fmt.Errorf("only TCP or unix socket "+
			"addresses are supported: %s", parsedAddr)

	default:
		// We'll now possibly apply the default port, use the local
		// host short circuit, or parse out an all interfaces listen.
		addrWithPort := verifyPort(strAddress, defaultPort)
		rawHost, rawPort, _ := net.SplitHostPort(addrWithPort)

		// If we reach this point, then we'll check to see if we have
		// an onion addresses, if so, we can directly pass the raw
		// address and port to create the proper address.
		if tor.IsOnionHost(rawHost) {
			portNum, err := strconv.Atoi(rawPort)
			if err != nil {
				return nil, err
			}

			return &tor.OnionAddr{
				OnionService: rawHost,
				Port:         portNum,
			}, nil
		}

		// Otherwise, we'll attempt the resolve the host. The Tor
		// resolver is unable to resolve local addresses,
		// IPv6 addresses, or the all-interfaces address, so we'll use
		// the system resolver instead for those.
		if rawHost == "" || IsLoopback(rawHost) ||
			isIPv6Host(rawHost) || isUnspecifiedHost(rawHost) {

			return net.ResolveTCPAddr("tcp", addrWithPort)
		}

		// If we've reached this point, then it's possible that this
		// resolve returns an error if it isn't able to resolve the
		// host. For example, local entries in /etc/hosts will fail to
		// be resolved by Tor. In order to handle this case, we'll fall
		// back to the normal system resolver if we fail with an
		// identifiable error.
		addr, err := tcpResolver("tcp", addrWithPort)
		if err != nil {
			torErrStr := "tor host is unreachable"
			if strings.Contains(err.Error(), torErrStr) {
				return net.ResolveTCPAddr("tcp", addrWithPort)
			}

			return nil, err
		}

		return addr, nil
	}
}

// ParseLNAddressString converts a string of the form <pubkey>@<addr> into an
// lnwire.NetAddress. The <pubkey> must be presented in hex, and result in a
// 33-byte, compressed public key that lies on the secp256k1 curve. The <addr>
// may be any address supported by ParseAddressString. If no port is specified,
// the defaultPort will be used. Any tcp addresses that need resolving will be
// resolved using the custom TCPResolver.
func ParseLNAddressString(strAddress string, defaultPort string,
	tcpResolver TCPResolver) (*lnwire.NetAddress, error) {

	// Split the address string around the @ sign.
	parts := strings.Split(strAddress, "@")

	// The string is malformed if there are not exactly two parts.
	if len(parts) != 2 {
		return nil, fmt.Errorf("invalid lightning address %s: "+
			"must be of the form <pubkey-hex>@<addr>", strAddress)
	}

	// Now, take the first portion as the hex pubkey, and the latter as the
	// address string.
	parsedPubKey, parsedAddr := parts[0], parts[1]

	// Decode the hex pubkey to get the raw compressed pubkey bytes.
	pubKeyBytes, err := hex.DecodeString(parsedPubKey)
	if err != nil {
		return nil, fmt.Errorf("invalid lightning address pubkey: %v", err)
	}

	// The compressed pubkey should have a length of exactly 33 bytes.
	if len(pubKeyBytes) != 33 {
		return nil, fmt.Errorf("invalid lightning address pubkey: "+
			"length must be 33 bytes, found %d", len(pubKeyBytes))
	}

	// Parse the pubkey bytes to verify that it corresponds to valid public
	// key on the secp256k1 curve.
	pubKey, err := btcec.ParsePubKey(pubKeyBytes)
	if err != nil {
		return nil, fmt.Errorf("invalid lightning address pubkey: %v", err)
	}

	// Finally, parse the address string using our generic address parser.
	addr, err := ParseAddressString(parsedAddr, defaultPort, tcpResolver)
	if err != nil {
		return nil, fmt.Errorf("invalid lightning address address: %v", err)
	}

	return &lnwire.NetAddress{
		IdentityKey: pubKey,
		Address:     addr,
	}, nil
}

// verifyPort makes sure that an address string has both a host and a port. If
// there is no port found, the default port is appended. If the address is just
// a port, then we'll assume that the user is using the short cut to specify a
// localhost:port address.
func verifyPort(address string, defaultPort string) string {
	host, port, err := net.SplitHostPort(address)
	if err != nil {
		// If the address itself is just an integer, then we'll assume
		// that we're mapping this directly to a localhost:port pair.
		// This ensures we maintain the legacy behavior.
		if _, err := strconv.Atoi(address); err == nil {
			return net.JoinHostPort("localhost", address)
		}

		// Otherwise, we'll assume that the address just failed to
		// attach its own port, so we'll use the default port. In the
		// case of IPv6 addresses, if the host is already surrounded by
		// brackets, then we'll avoid using the JoinHostPort function,
		// since it will always add a pair of brackets.
		if strings.HasPrefix(address, "[") {
			return address + ":" + defaultPort
		}
		return net.JoinHostPort(address, defaultPort)
	}

	// In the case that both the host and port are empty, we'll use the
	// default port.
	if host == "" && port == "" {
		return ":" + defaultPort
	}

	return address
}

// ClientAddressDialer creates a gRPC dialer that can also dial unix socket
// addresses instead of just TCP addresses.
func ClientAddressDialer(defaultPort string) func(context.Context,
	string) (net.Conn, error) {

	return func(ctx context.Context, addr string) (net.Conn, error) {
		parsedAddr, err := ParseAddressString(
			addr, defaultPort, net.ResolveTCPAddr,
		)
		if err != nil {
			return nil, err
		}

		d := net.Dialer{}
		return d.DialContext(
			ctx, parsedAddr.Network(), parsedAddr.String(),
		)
	}
}
<|endoftext|>


<commit_before>package linter

import (
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/skeema/skeema/fs"
	"github.com/skeema/tengo"
)

// A Detector function analyzes a schema for a particular problem, returning
// annotations for cases of the problem found.
type Detector func(*tengo.Schema, *fs.LogicalSchema, Options) []*Annotation

var problems map[string]Detector

// RegisterProblem adds a new named problem, along with its detector function.
func RegisterProblem(name string, fn Detector) {
	problems[name] = fn
}

func init() {
	problems = map[string]Detector{
		"no-pk":       noPKDetector,
		"bad-charset": badCharsetDetector,
		"bad-engine":  badEngineDetector,
	}
}

func noPKDetector(schema *tengo.Schema, logicalSchema *fs.LogicalSchema, _ Options) []*Annotation {
	results := make([]*Annotation, 0)
	for _, table := range schema.Tables {
		if table.PrimaryKey == nil {
			key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
			results = append(results, &Annotation{
				Statement: logicalSchema.Creates[key],
				Summary:   "No primary key",
				Message:   fmt.Sprintf("Table %s does not define a PRIMARY KEY", table.Name),
			})
		}
	}
	return results
}

func badCharsetDetector(schema *tengo.Schema, logicalSchema *fs.LogicalSchema, opts Options) []*Annotation {
	results := make([]*Annotation, 0)
	for _, table := range schema.Tables {
		// Check the table's default charset
		if !isAllowed(table.CharSet, opts.AllowedCharSets) {
			key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
			stmt := logicalSchema.Creates[key]
			re := regexp.MustCompile(fmt.Sprintf(`(?i)(default)?\s*(character\s+set|charset|collate)\s*=?\s*(%s|%s)`, table.CharSet, table.Collation))
			results = append(results, &Annotation{
				Statement:  logicalSchema.Creates[key],
				LineOffset: findLastLineOffset(re, stmt.Text),
				Summary:    "Character set not permitted",
				Message:    fmt.Sprintf("Table %s is using default character set %s, which is not listed in option allow-charset", table.Name, table.CharSet),
			})
			continue // if a table's default charset isn't allowed, don't generate col-level annotations too
		}

		// If default charset was ok, now check individual columns
		for _, col := range table.Columns {
			if col.CharSet != "" && !isAllowed(col.CharSet, opts.AllowedCharSets) {
				key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
				stmt := logicalSchema.Creates[key]
				re := regexp.MustCompile(fmt.Sprintf(`(?i)(character\s+set|charset|collate)\s*(%s|%s)`, col.CharSet, col.Collation))
				results = append(results, &Annotation{
					Statement:  logicalSchema.Creates[key],
					LineOffset: findFirstLineOffset(re, stmt.Text),
					Summary:    "Character set not permitted",
					Message:    fmt.Sprintf("Column %s of table %s is using character set %s, which is not listed in option allow-charset", col.Name, table.Name, col.CharSet),
				})
				break // stop after the first disallowed charset col per table
			}
		}
	}
	return results
}

func badEngineDetector(schema *tengo.Schema, logicalSchema *fs.LogicalSchema, opts Options) []*Annotation {
	results := make([]*Annotation, 0)
	for _, table := range schema.Tables {
		if !isAllowed(table.Engine, opts.AllowedEngines) {
			key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
			stmt := logicalSchema.Creates[key]
			re := regexp.MustCompile(fmt.Sprintf(`(?i)ENGINE\s*=?\s*%s`, table.Engine))
			results = append(results, &Annotation{
				Statement:  stmt,
				LineOffset: findFirstLineOffset(re, stmt.Text),
				Summary:    "Storage engine not permitted",
				Message:    fmt.Sprintf("Table %s is using storage engine %s, which is not listed in option allow-engine", table.Name, table.Engine),
			})
		}
	}

	return results
}

func problemExists(name string) bool {
	_, ok := problems[strings.ToLower(name)]
	return ok
}

func allProblemNames() []string {
	result := make([]string, 0, len(problems))
	for name := range problems {
		result = append(result, name)
	}
	sort.Strings(result)
	return result
}

// isAllowed performs a case-insensitive search for value in allowed, returning
// true if found.
func isAllowed(value string, allowed []string) bool {
	value = strings.ToLower(value)
	for _, allowedValue := range allowed {
		if value == strings.ToLower(allowedValue) {
			return true
		}
	}
	return false
}

// findFirstLineOffset returns the line offset (i.e. line number starting at 0)
// for the first match of re within createStatement. If no match occurs, 0 is
// returned. This may happen often due to createStatement being arbitrarily
// formatted.
func findFirstLineOffset(re *regexp.Regexp, createStatement string) int {
	loc := re.FindStringIndex(createStatement)
	if loc == nil {
		return 0
	}
	// Count how many newlines occur in createStatement before the match
	return strings.Count(createStatement[0:loc[0]], "\n")
}

// findLastLineOffset returns the line offset (i.e. line number starting at 0)
// for the last match of re within createStatement. If no match occurs, 0 is
// returned. This may happen often due to createStatement being arbitrarily
// formatted.
func findLastLineOffset(re *regexp.Regexp, createStatement string) int {
	locs := re.FindAllStringIndex(createStatement, -1)
	if locs == nil {
		return 0
	}
	lastLoc := locs[len(locs)-1]
	return strings.Count(createStatement[0:lastLoc[0]], "\n")
}
<commit_msg>lint: make annotation text more helpful<commit_after>package linter

import (
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/skeema/skeema/fs"
	"github.com/skeema/tengo"
)

// A Detector function analyzes a schema for a particular problem, returning
// annotations for cases of the problem found.
type Detector func(*tengo.Schema, *fs.LogicalSchema, Options) []*Annotation

var problems map[string]Detector

// RegisterProblem adds a new named problem, along with its detector function.
func RegisterProblem(name string, fn Detector) {
	problems[name] = fn
}

func init() {
	problems = map[string]Detector{
		"no-pk":       noPKDetector,
		"bad-charset": badCharsetDetector,
		"bad-engine":  badEngineDetector,
	}
}

func noPKDetector(schema *tengo.Schema, logicalSchema *fs.LogicalSchema, _ Options) []*Annotation {
	results := make([]*Annotation, 0)
	for _, table := range schema.Tables {
		if table.PrimaryKey == nil {
			key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
			message := fmt.Sprintf("Table %s does not define a PRIMARY KEY.", table.Name)
			if table.Engine == "InnoDB" && table.ClusteredIndexKey() == nil {
				message += " Lack of an explicit PRIMARY KEY hurts performance, and prevents use of third-party tools such as pt-online-schema-change."
			}
			results = append(results, &Annotation{
				Statement: logicalSchema.Creates[key],
				Summary:   "No primary key",
				Message:   message,
			})
		}
	}
	return results
}

func badCharsetDetector(schema *tengo.Schema, logicalSchema *fs.LogicalSchema, opts Options) []*Annotation {
	results := make([]*Annotation, 0)
	makeMessage := func(table *tengo.Table, column *tengo.Column) string {
		var subject, charSet, allowedList, moreInfo string
		if column == nil {
			subject = fmt.Sprintf("Table %s", table.Name)
			charSet = table.CharSet
		} else {
			subject = fmt.Sprintf("Column %s of table %s", column.Name, table.Name)
			charSet = column.CharSet
		}
		if len(opts.AllowedCharSets) == 1 {
			allowedList = fmt.Sprintf(" Only the %s character set is permitted.", opts.AllowedCharSets[0])
		} else if len(opts.AllowedCharSets) > 1 && len(opts.AllowedCharSets) <= 5 {
			allowedList = fmt.Sprintf(" The following character sets are permitted: %s.", strings.Join(opts.AllowedCharSets, ", "))
		}
		if charSet == "utf8" && isAllowed("utf8mb4", opts.AllowedCharSets) {
			moreInfo = "\nTo permit storage of all valid UTF-8 characters, use the utf8mb4 character set instead of the legacy utf8 character set."
		} else if charSet == "binary" {
			moreInfo = "\nUsing equivalent binary column types (e.g. BINARY, VARBINARY, BLOB) is preferred for readability."
		}
		return fmt.Sprintf("%s is using default character set %s, which is not listed in option allow-charset.%s%s", subject, charSet, allowedList, moreInfo)
	}

	for _, table := range schema.Tables {
		// Check the table's default charset
		if !isAllowed(table.CharSet, opts.AllowedCharSets) {
			key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
			stmt := logicalSchema.Creates[key]
			re := regexp.MustCompile(fmt.Sprintf(`(?i)(default)?\s*(character\s+set|charset|collate)\s*=?\s*(%s|%s)`, table.CharSet, table.Collation))
			results = append(results, &Annotation{
				Statement:  logicalSchema.Creates[key],
				LineOffset: findLastLineOffset(re, stmt.Text),
				Summary:    "Character set not permitted",
				Message:    makeMessage(table, nil),
			})
			continue // if a table's default charset isn't allowed, don't generate col-level annotations too
		}

		// If default charset was ok, now check individual columns
		for _, col := range table.Columns {
			if col.CharSet != "" && !isAllowed(col.CharSet, opts.AllowedCharSets) {
				key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
				stmt := logicalSchema.Creates[key]
				re := regexp.MustCompile(fmt.Sprintf(`(?i)(character\s+set|charset|collate)\s*(%s|%s)`, col.CharSet, col.Collation))
				results = append(results, &Annotation{
					Statement:  logicalSchema.Creates[key],
					LineOffset: findFirstLineOffset(re, stmt.Text),
					Summary:    "Character set not permitted",
					Message:    makeMessage(table, col),
				})
			}
		}
	}
	return results
}

func badEngineDetector(schema *tengo.Schema, logicalSchema *fs.LogicalSchema, opts Options) []*Annotation {
	results := make([]*Annotation, 0)
	for _, table := range schema.Tables {
		if !isAllowed(table.Engine, opts.AllowedEngines) {
			key := tengo.ObjectKey{Type: tengo.ObjectTypeTable, Name: table.Name}
			stmt := logicalSchema.Creates[key]
			re := regexp.MustCompile(fmt.Sprintf(`(?i)ENGINE\s*=?\s*%s`, table.Engine))
			message := fmt.Sprintf("Table %s is using storage engine %s, which is not listed in option allow-engine.", table.Name, table.Engine)
			if len(opts.AllowedEngines) == 1 {
				message = fmt.Sprintf("%s Only the %s storage engine is permitted.", message, opts.AllowedEngines[0])
			} else if len(opts.AllowedEngines) > 1 && len(opts.AllowedEngines) <= 5 {
				message = fmt.Sprintf("%s The following storage engines are permitted: %s.", message, strings.Join(opts.AllowedEngines, ", "))
			}
			results = append(results, &Annotation{
				Statement:  stmt,
				LineOffset: findFirstLineOffset(re, stmt.Text),
				Summary:    "Storage engine not permitted",
				Message:    message,
			})
		}
	}

	return results
}

func problemExists(name string) bool {
	_, ok := problems[strings.ToLower(name)]
	return ok
}

func allProblemNames() []string {
	result := make([]string, 0, len(problems))
	for name := range problems {
		result = append(result, name)
	}
	sort.Strings(result)
	return result
}

// isAllowed performs a case-insensitive search for value in allowed, returning
// true if found.
func isAllowed(value string, allowed []string) bool {
	value = strings.ToLower(value)
	for _, allowedValue := range allowed {
		if value == strings.ToLower(allowedValue) {
			return true
		}
	}
	return false
}

// findFirstLineOffset returns the line offset (i.e. line number starting at 0)
// for the first match of re within createStatement. If no match occurs, 0 is
// returned. This may happen often due to createStatement being arbitrarily
// formatted.
func findFirstLineOffset(re *regexp.Regexp, createStatement string) int {
	loc := re.FindStringIndex(createStatement)
	if loc == nil {
		return 0
	}
	// Count how many newlines occur in createStatement before the match
	return strings.Count(createStatement[0:loc[0]], "\n")
}

// findLastLineOffset returns the line offset (i.e. line number starting at 0)
// for the last match of re within createStatement. If no match occurs, 0 is
// returned. This may happen often due to createStatement being arbitrarily
// formatted.
func findLastLineOffset(re *regexp.Regexp, createStatement string) int {
	locs := re.FindAllStringIndex(createStatement, -1)
	if locs == nil {
		return 0
	}
	lastLoc := locs[len(locs)-1]
	return strings.Count(createStatement[0:lastLoc[0]], "\n")
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package recon contains code for recon.
package recon

import (
	"context"
	"fmt"
	"sort"

	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/server/v1/propertyvalues"
	"github.com/datacommonsorg/mixer/internal/store"
	"github.com/datacommonsorg/mixer/internal/util"
	"golang.org/x/sync/errgroup"
	"googlemaps.github.io/maps"
)

const (
	maxNumEntitiesPerRequest  = 5000
	maxMapsAPICallsInParallel = 25
)

type entityInfo struct {
	description string
	typ         string
}

// BulkFindEntities implements API for Mixer.BulkFindEntities.
func BulkFindEntities(
	ctx context.Context,
	in *pb.BulkFindEntitiesRequest,
	store *store.Store,
	mapsClient *maps.Client,
) (*pb.BulkFindEntitiesResponse, error) {
	if l := len(in.GetEntities()); l == 0 {
		return nil, fmt.Errorf("empty input")
	} else if l > maxNumEntitiesPerRequest {
		return nil, fmt.Errorf(
			"exceeded max number of entities per request (%d): %d",
			maxNumEntitiesPerRequest, l)
	}

	// Load input.
	entityInfoSet := map[entityInfo]struct{}{}
	for _, entity := range in.GetEntities() {
		description := entity.GetDescription()
		if description == "" {
			continue
		}
		entityInfoSet[entityInfo{description, entity.GetType()}] = struct{}{}
	}

	// Get place IDs.
	entityInfoToPlaceIDs, placeIDSet, err := resolvePlaceIDs(ctx, mapsClient, entityInfoSet)
	if err != nil {
		return nil, err
	}

	// Resolve place IDs to get DCIDs.
	placeIDToDCIDs, dcidSet, err := resolveDCIDs(ctx, placeIDSet, store)
	if err != nil {
		return nil, err
	}

	// Get types of the DCIDs.
	dcidToTypeSet, err := getPlaceTypes(ctx, dcidSet, store)
	if err != nil {
		return nil, err
	}

	// Assemble results.
	resp := &pb.BulkFindEntitiesResponse{}
	for entityInfo, placeIDs := range entityInfoToPlaceIDs {
		entity := &pb.BulkFindEntitiesResponse_Entity{
			Description: entityInfo.description,
			Type:        entityInfo.typ,
		}

		allDCIDs := []string{}
		for _, placeID := range placeIDs {
			if dcids, ok := placeIDToDCIDs[placeID]; ok {
				allDCIDs = append(allDCIDs, dcids...)
			}
		}

		if len(allDCIDs) != 0 {
			if entityInfo.typ == "" {
				// No type filtering.
				entity.Dcids = allDCIDs
			} else {
				// Type filtering.
				filteredDCIDs := []string{}
				for _, dcid := range allDCIDs {
					typeSet, ok := dcidToTypeSet[dcid]
					if !ok {
						continue
					}
					if _, ok := typeSet[entityInfo.typ]; ok {
						filteredDCIDs = append(filteredDCIDs, dcid)
					}
				}
				entity.Dcids = filteredDCIDs
			}
		}

		resp.Entities = append(resp.Entities, entity)
	}

	// Sort to make results determistic.
	sort.Slice(resp.Entities, func(i, j int) bool {
		if resp.Entities[i].GetDescription() == resp.Entities[j].GetDescription() {
			return resp.Entities[i].GetType() < resp.Entities[j].GetType()
		}
		return resp.Entities[i].GetDescription() < resp.Entities[j].GetDescription()
	})

	return resp, nil
}

func resolvePlaceIDs(
	ctx context.Context,
	mapsClient *maps.Client,
	entityInfoSet map[entityInfo]struct{},
) (
	map[entityInfo][]string, /* entityInfo -> [place ID] */
	map[string]struct{}, /* [place ID] for all entities */
	error,
) {
	type resolveResult struct {
		entityInfo *entityInfo
		placeIDs   []string
	}

	// Distribute entityInfoSet to maxMapsAPICallsInParallel shards for parallel processing.
	entityInfoListShards := make([][]entityInfo, maxMapsAPICallsInParallel)
	idx := 0
	for entityInfo := range entityInfoSet {
		entityInfoListShards[idx] = append(entityInfoListShards[idx], entityInfo)
		idx++
		if idx >= maxMapsAPICallsInParallel {
			idx = 0
		}
	}

	// The channel to receive results from parallel workers.
	resolveResultChan := make(chan resolveResult, len(entityInfoSet))

	// Worker function.
	mapsAPICallWorkerFunc := func(ctx context.Context, i int) func() error {
		return func() error {
			for _, entityInfo := range entityInfoListShards[i] {
				placeIDs, err := findPlaceIDsForEntity(ctx, mapsClient, &entityInfo)
				if err != nil {
					return err
				}
				// Only keep the first place ID, as the rest ones are usually much less accurate.
				resolveResultChan <- resolveResult{
					entityInfo: &entityInfo,
					placeIDs:   []string{placeIDs[0]}}
			}
			return nil
		}
	}

	// Call Maps API to find place IDs in parallel.
	// The errors in the Goroutines need to be captured, so we use errgroup.
	eg, errCtx := errgroup.WithContext(ctx)
	for i := 0; i < maxMapsAPICallsInParallel; i++ {
		eg.Go(mapsAPICallWorkerFunc(errCtx, i))
	}
	if err := eg.Wait(); err != nil {
		return nil, nil, err
	}
	close(resolveResultChan)

	// Read out the results sent by workers.
	entityInfoToPlaceIDs := map[entityInfo][]string{}
	placeIDSet := map[string]struct{}{}
	for res := range resolveResultChan {
		entityInfoToPlaceIDs[*res.entityInfo] = res.placeIDs
		for _, placeID := range res.placeIDs {
			placeIDSet[placeID] = struct{}{}
		}
	}

	return entityInfoToPlaceIDs, placeIDSet, nil
}

func findPlaceIDsForEntity(
	ctx context.Context,
	mapsClient *maps.Client,
	entityInfo *entityInfo,
) ([]string, error) {
	// When type is supplied, we append it to the description to increase the accuracy.
	input := entityInfo.description
	if t := entityInfo.typ; t != "" {
		input += (" " + t)
	}

	resp, err := mapsClient.FindPlaceFromText(ctx, &maps.FindPlaceFromTextRequest{
		Input:     input,
		InputType: maps.FindPlaceFromTextInputTypeTextQuery,
		Fields:    []maps.PlaceSearchFieldMask{maps.PlaceSearchFieldMaskPlaceID},
	})
	if err != nil {
		return nil, err
	}

	placeIDs := []string{}
	for _, candidate := range resp.Candidates {
		placeIDs = append(placeIDs, candidate.PlaceID)
	}

	return placeIDs, nil
}

func resolveDCIDs(
	ctx context.Context,
	placeIDSet map[string]struct{},
	store *store.Store,
) (
	map[string][]string, /* Place ID -> [DCID] */
	map[string]struct{}, /* [DCID] for all place IDs */
	error,
) {
	resolveResp, err := ResolveIds(ctx,
		&pb.ResolveIdsRequest{
			InProp:  "placeId",
			OutProp: "dcid",
			Ids:     util.StringSetToSlice(placeIDSet),
		},
		store)
	if err != nil {
		return nil, nil, err
	}
	placeIDToDCIDs := map[string][]string{}
	dcidSet := map[string]struct{}{}
	for _, entity := range resolveResp.GetEntities() {
		dcids := entity.GetOutIds()

		// Sort to make the result deterministic.
		sort.Strings(dcids)

		placeIDToDCIDs[entity.GetInId()] = append(placeIDToDCIDs[entity.GetInId()],
			dcids...)
		for _, dcid := range dcids {
			dcidSet[dcid] = struct{}{}
		}
	}
	return placeIDToDCIDs, dcidSet, nil
}

func getPlaceTypes(
	ctx context.Context,
	dcidSet map[string]struct{},
	store *store.Store,
) (
	map[string]map[string]struct{}, /* DCID -> {type} */
	error,
) {
	resp, err := propertyvalues.BulkPropertyValues(ctx,
		&pb.BulkPropertyValuesRequest{
			Property:  "typeOf",
			Nodes:     util.StringSetToSlice(dcidSet),
			Direction: util.DirectionOut,
		},
		store)
	if err != nil {
		return nil, err
	}
	dcidToTypeSet := map[string]map[string]struct{}{}
	for _, nodeInfo := range resp.GetData() {
		dcidToTypeSet[nodeInfo.GetNode()] = map[string]struct{}{}
		for _, entityInfo := range nodeInfo.GetValues() {
			dcidToTypeSet[nodeInfo.GetNode()][entityInfo.GetDcid()] = struct{}{}
		}
	}
	return dcidToTypeSet, nil
}
<commit_msg>Fix failing recon API test (#976)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package recon contains code for recon.
package recon

import (
	"context"
	"fmt"
	"sort"

	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/server/v1/propertyvalues"
	"github.com/datacommonsorg/mixer/internal/store"
	"github.com/datacommonsorg/mixer/internal/util"
	"golang.org/x/sync/errgroup"
	"googlemaps.github.io/maps"
)

const (
	maxNumEntitiesPerRequest  = 5000
	maxMapsAPICallsInParallel = 25
)

type entityInfo struct {
	description string
	typ         string
}

// BulkFindEntities implements API for Mixer.BulkFindEntities.
func BulkFindEntities(
	ctx context.Context,
	in *pb.BulkFindEntitiesRequest,
	store *store.Store,
	mapsClient *maps.Client,
) (*pb.BulkFindEntitiesResponse, error) {
	if l := len(in.GetEntities()); l == 0 {
		return nil, fmt.Errorf("empty input")
	} else if l > maxNumEntitiesPerRequest {
		return nil, fmt.Errorf(
			"exceeded max number of entities per request (%d): %d",
			maxNumEntitiesPerRequest, l)
	}

	// Load input.
	entityInfoSet := map[entityInfo]struct{}{}
	for _, entity := range in.GetEntities() {
		description := entity.GetDescription()
		if description == "" {
			continue
		}
		entityInfoSet[entityInfo{description, entity.GetType()}] = struct{}{}
	}

	// Get place IDs.
	entityInfoToPlaceIDs, placeIDSet, err := resolvePlaceIDs(ctx, mapsClient, entityInfoSet)
	if err != nil {
		return nil, err
	}

	// Resolve place IDs to get DCIDs.
	placeIDToDCIDs, dcidSet, err := resolveDCIDs(ctx, placeIDSet, store)
	if err != nil {
		return nil, err
	}

	// Get types of the DCIDs.
	dcidToTypeSet, err := getPlaceTypes(ctx, dcidSet, store)
	if err != nil {
		return nil, err
	}

	// Assemble results.
	resp := &pb.BulkFindEntitiesResponse{}
	for entityInfo, placeIDs := range entityInfoToPlaceIDs {
		entity := &pb.BulkFindEntitiesResponse_Entity{
			Description: entityInfo.description,
			Type:        entityInfo.typ,
		}

		allDCIDs := []string{}
		for _, placeID := range placeIDs {
			if dcids, ok := placeIDToDCIDs[placeID]; ok {
				allDCIDs = append(allDCIDs, dcids...)
			}
		}

		if len(allDCIDs) != 0 {
			if entityInfo.typ == "" {
				// No type filtering.
				entity.Dcids = allDCIDs
			} else {
				// Type filtering.
				filteredDCIDs := []string{}
				for _, dcid := range allDCIDs {
					typeSet, ok := dcidToTypeSet[dcid]
					if !ok {
						continue
					}
					if _, ok := typeSet[entityInfo.typ]; ok {
						filteredDCIDs = append(filteredDCIDs, dcid)
					}
				}
				entity.Dcids = filteredDCIDs
			}
		}

		resp.Entities = append(resp.Entities, entity)
	}

	// Sort to make results determistic.
	sort.Slice(resp.Entities, func(i, j int) bool {
		if resp.Entities[i].GetDescription() == resp.Entities[j].GetDescription() {
			return resp.Entities[i].GetType() < resp.Entities[j].GetType()
		}
		return resp.Entities[i].GetDescription() < resp.Entities[j].GetDescription()
	})

	return resp, nil
}

func resolvePlaceIDs(
	ctx context.Context,
	mapsClient *maps.Client,
	entityInfoSet map[entityInfo]struct{},
) (
	map[entityInfo][]string, /* entityInfo -> [place ID] */
	map[string]struct{}, /* [place ID] for all entities */
	error,
) {
	type resolveResult struct {
		entityInfo *entityInfo
		placeIDs   []string
	}

	// Distribute entityInfoSet to maxMapsAPICallsInParallel shards for parallel processing.
	entityInfoListShards := make([][]entityInfo, maxMapsAPICallsInParallel)
	idx := 0
	for entityInfo := range entityInfoSet {
		entityInfoListShards[idx] = append(entityInfoListShards[idx], entityInfo)
		idx++
		if idx >= maxMapsAPICallsInParallel {
			idx = 0
		}
	}

	// The channel to receive results from parallel workers.
	resolveResultChan := make(chan resolveResult, len(entityInfoSet))

	// Worker function.
	mapsAPICallWorkerFunc := func(ctx context.Context, i int) func() error {
		return func() error {
			for _, entityInfo := range entityInfoListShards[i] {
				placeIDs, err := findPlaceIDsForEntity(ctx, mapsClient, &entityInfo)
				if err != nil {
					return err
				}
				usedPlaceIds := []string{}
				if len(placeIDs) > 0 {
					// Only keep the first place ID, as the rest ones are usually much less accurate.
					usedPlaceIds = []string{placeIDs[0]}
				}
				resolveResultChan <- resolveResult{
					entityInfo: &entityInfo,
					placeIDs:   usedPlaceIds}
			}
			return nil
		}
	}

	// Call Maps API to find place IDs in parallel.
	// The errors in the Goroutines need to be captured, so we use errgroup.
	eg, errCtx := errgroup.WithContext(ctx)
	for i := 0; i < maxMapsAPICallsInParallel; i++ {
		eg.Go(mapsAPICallWorkerFunc(errCtx, i))
	}
	if err := eg.Wait(); err != nil {
		return nil, nil, err
	}
	close(resolveResultChan)

	// Read out the results sent by workers.
	entityInfoToPlaceIDs := map[entityInfo][]string{}
	placeIDSet := map[string]struct{}{}
	for res := range resolveResultChan {
		entityInfoToPlaceIDs[*res.entityInfo] = res.placeIDs
		for _, placeID := range res.placeIDs {
			placeIDSet[placeID] = struct{}{}
		}
	}

	return entityInfoToPlaceIDs, placeIDSet, nil
}

func findPlaceIDsForEntity(
	ctx context.Context,
	mapsClient *maps.Client,
	entityInfo *entityInfo,
) ([]string, error) {
	// When type is supplied, we append it to the description to increase the accuracy.
	input := entityInfo.description
	if t := entityInfo.typ; t != "" {
		input += (" " + t)
	}

	resp, err := mapsClient.FindPlaceFromText(ctx, &maps.FindPlaceFromTextRequest{
		Input:     input,
		InputType: maps.FindPlaceFromTextInputTypeTextQuery,
		Fields:    []maps.PlaceSearchFieldMask{maps.PlaceSearchFieldMaskPlaceID},
	})
	if err != nil {
		return nil, err
	}

	placeIDs := []string{}
	for _, candidate := range resp.Candidates {
		placeIDs = append(placeIDs, candidate.PlaceID)
	}

	return placeIDs, nil
}

func resolveDCIDs(
	ctx context.Context,
	placeIDSet map[string]struct{},
	store *store.Store,
) (
	map[string][]string, /* Place ID -> [DCID] */
	map[string]struct{}, /* [DCID] for all place IDs */
	error,
) {
	resolveResp, err := ResolveIds(ctx,
		&pb.ResolveIdsRequest{
			InProp:  "placeId",
			OutProp: "dcid",
			Ids:     util.StringSetToSlice(placeIDSet),
		},
		store)
	if err != nil {
		return nil, nil, err
	}
	placeIDToDCIDs := map[string][]string{}
	dcidSet := map[string]struct{}{}
	for _, entity := range resolveResp.GetEntities() {
		dcids := entity.GetOutIds()

		// Sort to make the result deterministic.
		sort.Strings(dcids)

		placeIDToDCIDs[entity.GetInId()] = append(placeIDToDCIDs[entity.GetInId()],
			dcids...)
		for _, dcid := range dcids {
			dcidSet[dcid] = struct{}{}
		}
	}
	return placeIDToDCIDs, dcidSet, nil
}

func getPlaceTypes(
	ctx context.Context,
	dcidSet map[string]struct{},
	store *store.Store,
) (
	map[string]map[string]struct{}, /* DCID -> {type} */
	error,
) {
	resp, err := propertyvalues.BulkPropertyValues(ctx,
		&pb.BulkPropertyValuesRequest{
			Property:  "typeOf",
			Nodes:     util.StringSetToSlice(dcidSet),
			Direction: util.DirectionOut,
		},
		store)
	if err != nil {
		return nil, err
	}
	dcidToTypeSet := map[string]map[string]struct{}{}
	for _, nodeInfo := range resp.GetData() {
		dcidToTypeSet[nodeInfo.GetNode()] = map[string]struct{}{}
		for _, entityInfo := range nodeInfo.GetValues() {
			dcidToTypeSet[nodeInfo.GetNode()][entityInfo.GetDcid()] = struct{}{}
		}
	}
	return dcidToTypeSet, nil
}
<|endoftext|>


<commit_before>package store

import (
	"fmt"
	"github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	leveldberrors "github.com/syndtr/goleveldb/leveldb/errors"
	"path/filepath"
)

// LevelDB holds a datastore name and its leveldb instance
type LevelDB struct {
	Name     string
	database *leveldb.DB
}

// New instantiates and open a new LevelDB instance backed by a leveldb database. If the
// leveldb database doesn't exist, one is created
func NewLevelDB(name string, storagePath string) (ldb *LevelDB, err error) {
	// Expand '~' as the full home directory path if appropriate
	path, err := homedir.Expand(storagePath)
	if err != nil {
		return nil, err
	}

	fullPath := filepath.Join(path, name)
	db, err := leveldb.OpenFile(fullPath, nil)

	if _, ok := err.(*leveldberrors.ErrCorrupted); ok {
		return nil, errors.Wrap(err, fmt.Sprintf("leveldb corrupted. Consider deleting [%s] and restarting if you don't mind losing data", fullPath))
	} else if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("failed to open file with path [%s]", fullPath))
	}

	return &LevelDB{name, db}, nil
}

// Close closes the LevelDB
func (ldb *LevelDB) Close() (err error) {
	return ldb.database.Close()
}

// GetString retrieves a value associated to the key
func (ldb *LevelDB) GetString(key string) (value string, err error) {
	val, err := ldb.Get([]byte(key))

	return string(val), err
}

// Get retrieves a value associated to the key
func (ldb *LevelDB) Get(key []byte) (value []byte, err error) {
	value, err = ldb.database.Get(key, nil)
	if err != nil {
		return nil, err
	}

	return value, nil
}

// PutString adds or updates a value associated to the key
func (ldb *LevelDB) PutString(key string, value string) (err error) {
	return ldb.database.Put([]byte(key), []byte(value), nil)
}

// Put adds or updates a value associated to the key
func (ldb *LevelDB) Put(key []byte, value []byte) (err error) {
	return ldb.database.Put(key, value, nil)
}

// Scan returns the complete set of key/values from the database
func (ldb *LevelDB) Scan() (entries map[string]string, err error) {
	entries = map[string]string{}
	iter := ldb.database.NewIterator(nil, nil)
	for iter.Next() {
		key := string(iter.Key())
		value := string(iter.Value())
		entries[key] = value
	}

	iter.Release()
	err = iter.Error()

	return entries, err
}
<commit_msg>Fix New -> NewLevelDB in godoc<commit_after>package store

import (
	"fmt"
	"github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	leveldberrors "github.com/syndtr/goleveldb/leveldb/errors"
	"path/filepath"
)

// LevelDB holds a datastore name and its leveldb instance
type LevelDB struct {
	Name     string
	database *leveldb.DB
}

// NewLevelDB instantiates and open a new LevelDB instance backed by a leveldb database. If the
// leveldb database doesn't exist, one is created
func NewLevelDB(name string, storagePath string) (ldb *LevelDB, err error) {
	// Expand '~' as the full home directory path if appropriate
	path, err := homedir.Expand(storagePath)
	if err != nil {
		return nil, err
	}

	fullPath := filepath.Join(path, name)
	db, err := leveldb.OpenFile(fullPath, nil)

	if _, ok := err.(*leveldberrors.ErrCorrupted); ok {
		return nil, errors.Wrap(err, fmt.Sprintf("leveldb corrupted. Consider deleting [%s] and restarting if you don't mind losing data", fullPath))
	} else if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("failed to open file with path [%s]", fullPath))
	}

	return &LevelDB{name, db}, nil
}

// Close closes the LevelDB
func (ldb *LevelDB) Close() (err error) {
	return ldb.database.Close()
}

// GetString retrieves a value associated to the key
func (ldb *LevelDB) GetString(key string) (value string, err error) {
	val, err := ldb.Get([]byte(key))

	return string(val), err
}

// Get retrieves a value associated to the key
func (ldb *LevelDB) Get(key []byte) (value []byte, err error) {
	value, err = ldb.database.Get(key, nil)
	if err != nil {
		return nil, err
	}

	return value, nil
}

// PutString adds or updates a value associated to the key
func (ldb *LevelDB) PutString(key string, value string) (err error) {
	return ldb.database.Put([]byte(key), []byte(value), nil)
}

// Put adds or updates a value associated to the key
func (ldb *LevelDB) Put(key []byte, value []byte) (err error) {
	return ldb.database.Put(key, value, nil)
}

// Scan returns the complete set of key/values from the database
func (ldb *LevelDB) Scan() (entries map[string]string, err error) {
	entries = map[string]string{}
	iter := ldb.database.NewIterator(nil, nil)
	for iter.Next() {
		key := string(iter.Key())
		value := string(iter.Value())
		entries[key] = value
	}

	iter.Release()
	err = iter.Error()

	return entries, err
}
<|endoftext|>


<commit_before>package datakit

import (
	"context"
	"fmt"
	"log"
	"testing"
)

func TestConfig(t *testing.T) {
	ctx := context.Background()
	log.Println("Testing the configuration interface")

	client, err := dial(ctx)
	if err != nil {
		t.Fatalf("Failed to connect to db: %v", err)
	}
	r, err := NewRecord(ctx, client, []string{"master", "defaults"}, "defaults", "state", []string{"tests"})
	if err != nil {
		t.Fatalf("NewRecord failed: %v", err)
	}
	err = write(ctx, client, []string{"tests", "name"}, "hello")
	err = write(ctx, client, []string{"tests", "ncpu"}, "1")
	err = write(ctx, client, []string{"tests", "running"}, "true")
	r.Wait(ctx)

	nameF := r.StringField("name", "hello")
	ncpuF := r.IntField("ncpu", 1)
	runningF := r.BoolField("running", true)

	name, nameV := nameF.Get()
	ncpu, ncpuV := ncpuF.Get()
	running, runningV := runningF.Get()
	fmt.Printf("name: %s, ncpu: %d, running: %t\n", name, ncpu, running)

	if nameF.HasChanged(nameV) {
		t.Fatalf("name has unexpectedly changed")
	}
	if ncpuF.HasChanged(ncpuV) {
		t.Fatalf("ncpu has unexpectedly changed")
	}
	if runningF.HasChanged(runningV) {
		t.Fatalf("running has unexpectedly changed")
	}
	err = write(ctx, client, []string{"tests", "name"}, "there")
	if err != nil {
		t.Fatalf("failed to write new name value: %v", err)
	}
	r.Wait(ctx)
	if nameF.HasChanged(nameV) {
		name, nameV = nameF.Get()
		fmt.Printf("name has changed to %s\n", name)
	} else {
		t.Fatalf("name should have changed but hasn't")
	}
	// ncpu should not have changed
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		t.Fatalf("ncpu has unexpectedly changed to %d\n", ncpu)
	}
	if runningF.HasChanged(runningV) {
		t.Fatalf("running has unexpectedly changed")
	}
	err = write(ctx, client, []string{"tests", "ncpu"}, "5")
	if err != nil {
		t.Fatalf("failed to write new ncpu value: %v", err)
	}
	r.Wait(ctx)
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		fmt.Printf("ncpu has changed to %d\n", ncpu)
	}
	err = write(ctx, client, []string{"tests", "running"}, "rubbish")
	if err != nil {
		t.Fatalf("failed to write new running value: %v", err)
	}
	r.Wait(ctx)
	if runningF.HasChanged(runningV) {
		running, runningV = runningF.Get()
		fmt.Printf("running has changed to %t\n", running)
	}
	// Schema upgrade testing:
	// 1. no change to ncpus (current value is 5)
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		t.Fatalf("ncpu has unexpectedly changed to %d\n", ncpu)
	}
	// 2. a no-op upgrade
	r.Upgrade(ctx, 1)
	if ncpuF.HasChanged(ncpuV) {
		t.Fatalf("ncpu has unexpectedly changed")
	}
	// 3. a real upgrade
	r.Upgrade(ctx, 2)
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		fmt.Printf("ncpu has changed to %d\n", ncpu)
		if ncpu != 1 {
			t.Fatalf("Upgrade didn't set ncpu to 1\n")
		}
	}
}

func write(ctx context.Context, client *Client, path []string, value string) error {
	t, err := NewTransaction(ctx, client, "master", "test-tmp")

	if err != nil {
		return err
	}
	err = t.Write(ctx, path, value)
	if err != nil {
		return err
	}
	err = t.Commit(ctx, "Write test")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>go: add a config test that defaults are updated<commit_after>package datakit

import (
	"context"
	"fmt"
	"log"
	"testing"
)

func TestConfig(t *testing.T) {
	ctx := context.Background()
	log.Println("Testing the configuration interface")

	client, err := dial(ctx)
	if err != nil {
		t.Fatalf("Failed to connect to db: %v", err)
	}
	r, err := NewRecord(ctx, client, []string{"master", "defaults"}, "defaults", "state", []string{"tests"})
	if err != nil {
		t.Fatalf("NewRecord failed: %v", err)
	}
	err = write(ctx, client, []string{"tests", "name"}, "hello")
	err = write(ctx, client, []string{"tests", "ncpu"}, "1")
	err = write(ctx, client, []string{"tests", "running"}, "true")
	r.Wait(ctx)

	nameF := r.StringField("name", "hello")
	ncpuF := r.IntField("ncpu", 1)
	runningF := r.BoolField("running", true)

	name, nameV := nameF.Get()
	ncpu, ncpuV := ncpuF.Get()
	running, runningV := runningF.Get()
	fmt.Printf("name: %s, ncpu: %d, running: %t\n", name, ncpu, running)

	if nameF.HasChanged(nameV) {
		t.Fatalf("name has unexpectedly changed")
	}
	if ncpuF.HasChanged(ncpuV) {
		t.Fatalf("ncpu has unexpectedly changed")
	}
	if runningF.HasChanged(runningV) {
		t.Fatalf("running has unexpectedly changed")
	}
	err = write(ctx, client, []string{"tests", "name"}, "there")
	if err != nil {
		t.Fatalf("failed to write new name value: %v", err)
	}
	r.Wait(ctx)
	if nameF.HasChanged(nameV) {
		name, nameV = nameF.Get()
		fmt.Printf("name has changed to %s\n", name)
	} else {
		t.Fatalf("name should have changed but hasn't")
	}
	// ncpu should not have changed
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		t.Fatalf("ncpu has unexpectedly changed to %d\n", ncpu)
	}
	if runningF.HasChanged(runningV) {
		t.Fatalf("running has unexpectedly changed")
	}
	err = write(ctx, client, []string{"tests", "ncpu"}, "5")
	if err != nil {
		t.Fatalf("failed to write new ncpu value: %v", err)
	}
	r.Wait(ctx)
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		fmt.Printf("ncpu has changed to %d\n", ncpu)
	}
	err = write(ctx, client, []string{"tests", "running"}, "rubbish")
	if err != nil {
		t.Fatalf("failed to write new running value: %v", err)
	}
	r.Wait(ctx)
	if runningF.HasChanged(runningV) {
		running, runningV = runningF.Get()
		fmt.Printf("running has changed to %t\n", running)
	}
	// Schema upgrade testing:
	// 1. no change to ncpus (current value is 5)
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		t.Fatalf("ncpu has unexpectedly changed to %d\n", ncpu)
	}
	// 2. a no-op upgrade
	r.Upgrade(ctx, 1)
	if ncpuF.HasChanged(ncpuV) {
		t.Fatalf("ncpu has unexpectedly changed")
	}
	// 3. a real upgrade
	r.Upgrade(ctx, 2)
	if ncpuF.HasChanged(ncpuV) {
		ncpu, ncpuV = ncpuF.Get()
		fmt.Printf("ncpu has changed to %d\n", ncpu)
		if ncpu != 1 {
			t.Fatalf("Upgrade didn't set ncpu to 1\n")
		}
	}
}

func TestChangeDefaults(t *testing.T) {
	ctx := context.Background()
	log.Println("Check that defaults can be updated")

	client, err := dial(ctx)
	if err != nil {
		t.Fatalf("Failed to connect to db: %v", err)
	}
	key := "change-default-key"
	if err = rm(ctx, client, "defaults", []string{"tests", key}); err != nil {
		t.Fatalf("Failed to remove previous test state")
	}

	r, err := NewRecord(ctx, client, []string{"master", "defaults"}, "defaults", "state", []string{"tests"})
	if err != nil {
		t.Fatalf("NewRecord failed: %v", err)
	}
	r.Wait(ctx)

	nameF := r.StringField(key, "hello")
	r.Seal(ctx)

	def, err := read(ctx, client, "defaults", []string{"tests", key})
	if err != nil {
		t.Fatalf("r.StringField didn't create a defaults branch entry")
	}
	if def != "hello" {
		t.Fatalf("r.StringField should have set the default value to hello: %s", def)
	}
	name, _ := nameF.Get()
	if name != "hello" {
		t.Fatalf("r.Get after r.StringField should have read the default value of hello: %s", name)
	}

	r, err = NewRecord(ctx, client, []string{"master", "defaults"}, "defaults", "state", []string{"tests"})
	if err != nil {
		t.Fatalf("NewRecord failed: %v", err)
	}
	r.Wait(ctx)
	nameF = r.StringField(key, "there")
	r.Seal(ctx)

	def, err = read(ctx, client, "defaults", []string{"tests", key})
	if err != nil {
		t.Fatalf("r.StringField didn't create a defaults branch entry")
	}
	if def != "there" {
		t.Fatalf("r.StringField should have reset the default value to there: %s", def)
	}
	name, _ = nameF.Get()
	if name != "there" {
		t.Fatalf("r.Get after r.StringField should have read the new default value of there: %s", name)
	}
}

func write(ctx context.Context, client *Client, path []string, value string) error {
	t, err := NewTransaction(ctx, client, "master", "test-tmp")

	if err != nil {
		return err
	}
	err = t.Write(ctx, path, value)
	if err != nil {
		return err
	}
	err = t.Commit(ctx, "Write test")
	if err != nil {
		return err
	}

	return nil
}

func rm(ctx context.Context, client *Client, branch string, path []string) error {
	t, err := NewTransaction(ctx, client, branch, "test-tmp")

	if err != nil {
		return err
	}
	err = t.Remove(ctx, path)
	if err != nil {
		return err
	}
	err = t.Commit(ctx, "Write test")
	if err != nil {
		return err
	}

	return nil
}

func read(ctx context.Context, client *Client, branch string, path []string) (string, error) {
	sha, err := Head(ctx, client, branch)
	if err != nil {
		return "", err
	}
	snap := NewSnapshot(ctx, client, COMMIT, sha)
	return snap.Read(ctx, path)
}
<|endoftext|>


<commit_before>package rss

import (
	"connectordb/plugins/rest/restcore"
	"connectordb/streamdb"
	"connectordb/streamdb/operator"
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
)

//GetRSS gets an RSS feed of the given stream. It returns the smaller of 2 days or 500 datapoints
func GetRSS(o operator.Operator, writer http.ResponseWriter, request *http.Request, logger *log.Entry) error {
	_, _, _, streampath := restcore.GetStreamPath(request)

	tnow := time.Now()
	t2day := tnow.Add(-2 * 24 * time.Hour)

	s, err := o.ReadStream(streampath)
	if err != nil {
		restcore.WriteError(writer, logger, http.StatusForbidden, err, false)
		return err
	}

	dr, err := o.GetStreamTimeRange(streampath, float64(t2day.Unix()), float64(tnow.Unix()), int64(500))
	if err != nil {
		restcore.WriteError(writer, logger, http.StatusInternalServerError, err, true)
		return err
	}

	f := &feeds.Feed{
		Title:       s.Name,
		Created:     time.Now(),
		Description: s.Nickname,
		Link:        &feeds.Link{Href: "https://connectordb.com/"},
	}

	f.Items = make([]*feeds.Item, 0, 500)

	for dp, err := dr.Next(); err == nil && dp != nil; dp, err = dr.Next() {
		v, err := json.Marshal(dp)
		if err != nil {
			restcore.WriteError(writer, logger, http.StatusInternalServerError, err, true)
			return err
		}
		f.Items = append(f.Items, &feeds.Item{
			Created: time.Unix(0, int64(dp.Timestamp*1e9)),
			Link:    &feeds.Link{Href: ""},
			Title:   string(v),
		})
	}

	val, err := f.ToAtom()
	if err != nil {
		restcore.WriteError(writer, logger, http.StatusInternalServerError, err, true)
		return err
	}

	writer.Header().Set("Content-Length", strconv.Itoa(len([]byte(val))))
	writer.Header().Set("Content-Type", "application/xml; charset=utf-8")
	writer.WriteHeader(http.StatusOK)
	writer.Write([]byte(val))

	return nil
}

//Router returns a fully formed Gorilla router given an optional prefix
func Router(db *streamdb.Database, prefix *mux.Router) *mux.Router {
	if prefix == nil {
		prefix = mux.NewRouter()
	}

	//Allow for the application to match /path and /path/ to the same place.
	prefix.StrictSlash(true)

	prefix.HandleFunc("/{user}/{device}/{stream}", restcore.Authenticator(GetRSS, db)).Methods("GET")

	return prefix
}
<commit_msg>RSS works a bit better<commit_after>package rss

import (
	"connectordb/plugins/rest/restcore"
	"connectordb/streamdb"
	"connectordb/streamdb/operator"
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
)

//GetRSS gets an RSS feed of the given stream. It returns the smaller of 2 days or 500 datapoints
func GetRSS(o operator.Operator, writer http.ResponseWriter, request *http.Request, logger *log.Entry) error {
	_, _, _, streampath := restcore.GetStreamPath(request)

	tnow := time.Now()
	t2day := tnow.Add(-2 * 24 * time.Hour)

	s, err := o.ReadStream(streampath)
	if err != nil {
		restcore.WriteError(writer, logger, http.StatusForbidden, err, false)
		return err
	}

	dr, err := o.GetStreamTimeRange(streampath, float64(t2day.Unix()), float64(tnow.Unix()), int64(500))
	if err != nil {
		restcore.WriteError(writer, logger, http.StatusInternalServerError, err, true)
		return err
	}

	f := &feeds.Feed{
		Title:       s.Name,
		Created:     time.Now(),
		Description: s.Nickname,
		Link:        &feeds.Link{Href: "https://connectordb.com/"},
	}

	f.Items = make([]*feeds.Item, 0, 500)

	for dp, err := dr.Next(); err == nil && dp != nil; dp, err = dr.Next() {
		v, err := json.Marshal(dp.Data)
		if err != nil {
			restcore.WriteError(writer, logger, http.StatusInternalServerError, err, true)
			return err
		}
		f.Items = append(f.Items, &feeds.Item{
			Created: time.Unix(0, int64(dp.Timestamp*1e9)),
			Link:    &feeds.Link{Href: ""},
			Title:   string(v),
		})
	}

	val, err := f.ToAtom()
	if err != nil {
		restcore.WriteError(writer, logger, http.StatusInternalServerError, err, true)
		return err
	}

	writer.Header().Set("Content-Length", strconv.Itoa(len([]byte(val))))
	writer.Header().Set("Content-Type", "application/xml; charset=utf-8")
	writer.WriteHeader(http.StatusOK)
	writer.Write([]byte(val))

	return nil
}

//Router returns a fully formed Gorilla router given an optional prefix
func Router(db *streamdb.Database, prefix *mux.Router) *mux.Router {
	if prefix == nil {
		prefix = mux.NewRouter()
	}

	//Allow for the application to match /path and /path/ to the same place.
	prefix.StrictSlash(true)

	prefix.HandleFunc("/{user}/{device}/{stream}", restcore.Authenticator(GetRSS, db)).Methods("GET")

	return prefix
}
<|endoftext|>


<commit_before>package led

// Color represents 32-bit alpha-premultipled color, having 8 bits for each of
// alpha, red, green and blue (0xAARRGGBB).
type Color uint32

// RGBA returns Color for alpha-premultipled r, g, b (r<=a && g<=a && b<=a). To
// obtain Color from not alpha-premultipled R, G, B use RGB(R, G, B).Mask(a).
func RGBA(r, g, b, a byte) Color {
	return Color(a)<<24 | Color(r)<<16 | Color(g)<<8 | Color(b)
}

func RGB(r, g, b byte) Color {
	return 0xFF<<24 | Color(r)<<16 | Color(g)<<8 | Color(b)
}

func (c Color) Alpha() byte {
	return byte(c >> 24)
}

func (c Color) Red() byte {
	return byte(c >> 16)
}

func (c Color) Green() byte {
	return byte(c >> 8)
}

func (c Color) Blue() byte {
	return byte(c)
}

func (c Color) Mask(mask byte) Color {
	// Consider replace the original formula (c*m + 127) / 255 with cheaper
	// (c*m + 255) >> 8 which avoids division and (c*255 + 255) >> 8 == c.

	m := uint(mask)
	r := (uint(c.Red())*m + 127) / 255
	g := (uint(c.Green())*m + 127) / 255
	b := (uint(c.Blue())*m + 127) / 255
	a := (uint(c.Alpha())*m + 127) / 255

	return Color(a<<24 | r<<16 | g<<8 | b)
}

func (c1 Color) Blend(c2 Color, mask byte) Color {
	r1 := uint(c1.Red())
	g1 := uint(c1.Green())
	b1 := uint(c1.Blue())
	a1 := uint(c1.Alpha())
	r2 := uint(c2.Red())
	g2 := uint(c2.Green())
	b2 := uint(c2.Blue())
	a2 := uint(c2.Alpha())
	m2 := uint(mask)

	m1 := (255*255 - a2*m2)
	m2 *= 255
	r := (r1*m1 + r2*m2) / (255 * 255)
	g := (g1*m1 + g2*m2) / (255 * 255)
	b := (b1*m1 + b2*m2) / (255 * 255)
	a := (a1*m1 + a2*m2) / (255 * 255)

	return Color(a<<24 | r<<16 | g<<8 | b)
}
<commit_msg>led: Rounding to the nearest in Color.Blend.<commit_after>package led

// Color represents 32-bit alpha-premultipled color, having 8 bits for each of
// alpha, red, green and blue (0xAARRGGBB).
type Color uint32

// RGBA returns Color for alpha-premultipled r, g, b (r<=a && g<=a && b<=a). To
// obtain Color from not alpha-premultipled R, G, B use RGB(R, G, B).Mask(a).
func RGBA(r, g, b, a byte) Color {
	return Color(a)<<24 | Color(r)<<16 | Color(g)<<8 | Color(b)
}

func RGB(r, g, b byte) Color {
	return 0xFF<<24 | Color(r)<<16 | Color(g)<<8 | Color(b)
}

func (c Color) Alpha() byte {
	return byte(c >> 24)
}

func (c Color) Red() byte {
	return byte(c >> 16)
}

func (c Color) Green() byte {
	return byte(c >> 8)
}

func (c Color) Blue() byte {
	return byte(c)
}

func (c Color) Mask(mask byte) Color {
	// Consider replace the original formula (c*m + 127) / 255 with cheaper
	// (c*m + 255) >> 8 which avoids division and (c*255 + 255) >> 8 == c.

	m := uint(mask)
	r := (uint(c.Red())*m + 127) / 255
	g := (uint(c.Green())*m + 127) / 255
	b := (uint(c.Blue())*m + 127) / 255
	a := (uint(c.Alpha())*m + 127) / 255

	return Color(a<<24 | r<<16 | g<<8 | b)
}

func (c1 Color) Blend(c2 Color, mask byte) Color {
	r1 := uint(c1.Red())
	g1 := uint(c1.Green())
	b1 := uint(c1.Blue())
	a1 := uint(c1.Alpha())
	r2 := uint(c2.Red())
	g2 := uint(c2.Green())
	b2 := uint(c2.Blue())
	a2 := uint(c2.Alpha())
	m2 := uint(mask)

	m1 := (255*255 - a2*m2)
	m2 *= 255
	r := (r1*m1 + r2*m2 + 32512) / (255 * 255)
	g := (g1*m1 + g2*m2 + 32512) / (255 * 255)
	b := (b1*m1 + b2*m2 + 32512) / (255 * 255)
	a := (a1*m1 + a2*m2 + 32512) / (255 * 255)

	return Color(a<<24 | r<<16 | g<<8 | b)
}
<|endoftext|>