<commit_before>// Package camo provides an HTTP proxy server with content type
// restrictions as well as regex host allow list support.
package camo

import (
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/cactus/go-camo/camo/encoding"
	"github.com/cactus/gologit"
	"github.com/gorilla/mux"
	httpclient "github.com/mreiferson/go-httpclient"
)

// Config holds configuration data used when creating a Proxy with New.
type Config struct {
	// HMACKey is a byte slice to be used as the hmac key
	HMACKey []byte
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize int64
	// MaxRedirects is the maximum number of redirects to follow.
	MaxRedirects int
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout time.Duration
	// Server name used in Headers and Via checks
	ServerName string
	// Default headers to add to each response
	AddHeaders map[string]string
}

// ProxyMetrics interface for Proxy to use for stats/metrics.
// This must be goroutine safe, as AddBytes and AddServed will be called from
// many goroutines.
type ProxyMetrics interface {
	AddBytes(bc int64)
	AddServed()
}

// A Proxy is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow list support.
type Proxy struct {
	client *http.Client
	config *Config
	// compiled allow list regex
	allowList []*regexp.Regexp
	metrics   ProxyMetrics
}

// NotFound handles client requests that did not match a valid
// mux route.
func (p *Proxy) NotFoundHandler() http.Handler {
	handler := func(w http.ResponseWriter, req *http.Request) {
		h := w.Header()
		for k, v := range p.config.AddHeaders {
			h.Set(k, v)
		}
		h.Set("Server", p.config.ServerName)
		h.Set("Date", formattedDate.String())
		http.Error(w, "404 Not Found", 404)
	}
	return http.HandlerFunc(handler)
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	gologit.Debugln("Request:", req.URL)
	if p.metrics != nil {
		go p.metrics.AddServed()
	}

	// add default headers
	wh := w.Header()
	for k, v := range p.config.AddHeaders {
		wh.Set(k, v)
	}
	wh.Set("Server", p.config.ServerName)
	wh.Set("Date", formattedDate.String())

	if req.Header.Get("Via") == p.config.ServerName {
		http.Error(w, "Request loop failure", http.StatusNotFound)
		return
	}

	vars := mux.Vars(req)
	sURL, ok := encoding.DecodeURL(p.config.HMACKey, vars["sigHash"], vars["encodedURL"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	gologit.Debugln("URL:", sURL)
	gologit.Debugln("Client request:", req)

	u, err := url.Parse(sURL)
	if err != nil {
		gologit.Debugln("url parse error:", err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	u.Host = strings.ToLower(u.Host)
	if u.Host == "" || localhostRegex.MatchString(u.Host) {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out rfc1918 hosts
	ip := net.ParseIP(u.Host)
	if ip != nil {
		if addr1918PrefixRegex.MatchString(ip.String()) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest(req.Method, sURL, nil)
	if err != nil {
		gologit.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	if req.Header.Get("X-Forwarded-For") == "" {
		host, _, err := net.SplitHostPort(req.RemoteAddr)
		if err == nil && !addr1918PrefixRegex.MatchString(host) {
			nreq.Header.Add("X-Forwarded-For", host)
		}
	}

	// add an accept header if the client didn't send one
	if nreq.Header.Get("Accept") == "" {
		nreq.Header.Add("Accept", "image/*")
	}

	nreq.Header.Add("user-agent", p.config.ServerName)
	nreq.Header.Add("via", p.config.ServerName)

	gologit.Debugln("Built outgoing request:", nreq)

	resp, err := p.client.Do(nreq)
	if err != nil {
		gologit.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()
	gologit.Debugln("Response from upstream:", resp)

	// check for too large a response
	if resp.ContentLength > p.config.MaxSize {
		gologit.Debugln("Content length exceeded", sURL)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		if !strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
			gologit.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		gologit.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303, 307:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	w.WriteHeader(resp.StatusCode)

	// since this uses io.Copy from the respBody, it is streaming
	// from the request to the response. This means it will nearly
	// always end up with a chunked response.
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		opErr, ok := err.(*net.OpError)
		if ok {
			switch opErr.Err {
			case syscall.EPIPE:
				// broken pipe - endpoint terminated the conn
				// log as debug only.
				gologit.Debugln("OpError writing response:", err)
			case syscall.ECONNRESET:
				// connection reset by peer - endpoint terminated the conn
				// log as debug only.
				gologit.Debugln("OpError writing response:", err)
			default:
				// log anything else normally
				gologit.Println("OpError writing response:", err)
			}
		} else {
			// unknown error and not an OpError.
			gologit.Println("Error writing response:", err)
		}
		return
	}

	if p.metrics != nil {
		go p.metrics.AddBytes(bW)
	}
	gologit.Debugln("Response to client:", w)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *Proxy) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// SetMetricsCollector sets a proxy metrics (ProxyMetrics interface) for
// the proxy
func (p *Proxy) SetMetricsCollector(pm ProxyMetrics) {
	p.metrics = pm
}

// New returns a new Proxy. An error is returned if there was a failure
// to parse the regex from the passed Config.
func New(pc Config) (*Proxy, error) {
	tr := &httpclient.Transport{
		MaxIdleConnsPerHost: 8,
		ConnectTimeout:      2 * time.Second,
		RequestTimeout:      pc.RequestTimeout}

	// spawn an idle conn trimmer
	go func() {
		// prunes every 5 minutes. this is just a guess at an
		// initial value. very busy severs may want to lower this...
		for {
			time.Sleep(5 * time.Minute)
			tr.CloseIdleConnections()
		}
	}()

	client := &http.Client{Transport: tr}
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= pc.MaxRedirects {
			return errors.New("Too many redirects")
		}
		return nil
	}

	var allow []*regexp.Regexp
	var c *regexp.Regexp
	var err error
	// compile allow list
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &Proxy{
		client:    client,
		config:    &pc,
		allowList: allow}, nil
}
<commit_msg>handle proxy request errors a bit better<commit_after>// Package camo provides an HTTP proxy server with content type
// restrictions as well as regex host allow list support.
package camo

import (
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/cactus/go-camo/camo/encoding"
	"github.com/cactus/gologit"
	"github.com/gorilla/mux"
	httpclient "github.com/mreiferson/go-httpclient"
)

// Config holds configuration data used when creating a Proxy with New.
type Config struct {
	// HMACKey is a byte slice to be used as the hmac key
	HMACKey []byte
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize int64
	// MaxRedirects is the maximum number of redirects to follow.
	MaxRedirects int
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout time.Duration
	// Server name used in Headers and Via checks
	ServerName string
	// Default headers to add to each response
	AddHeaders map[string]string
}

// ProxyMetrics interface for Proxy to use for stats/metrics.
// This must be goroutine safe, as AddBytes and AddServed will be called from
// many goroutines.
type ProxyMetrics interface {
	AddBytes(bc int64)
	AddServed()
}

// A Proxy is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow list support.
type Proxy struct {
	client *http.Client
	config *Config
	// compiled allow list regex
	allowList []*regexp.Regexp
	metrics   ProxyMetrics
}

// NotFound handles client requests that did not match a valid
// mux route.
func (p *Proxy) NotFoundHandler() http.Handler {
	handler := func(w http.ResponseWriter, req *http.Request) {
		h := w.Header()
		for k, v := range p.config.AddHeaders {
			h.Set(k, v)
		}
		h.Set("Server", p.config.ServerName)
		h.Set("Date", formattedDate.String())
		http.Error(w, "404 Not Found", 404)
	}
	return http.HandlerFunc(handler)
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	gologit.Debugln("Request:", req.URL)
	if p.metrics != nil {
		go p.metrics.AddServed()
	}

	// add default headers
	wh := w.Header()
	for k, v := range p.config.AddHeaders {
		wh.Set(k, v)
	}
	wh.Set("Server", p.config.ServerName)
	wh.Set("Date", formattedDate.String())

	if req.Header.Get("Via") == p.config.ServerName {
		http.Error(w, "Request loop failure", http.StatusNotFound)
		return
	}

	vars := mux.Vars(req)
	sURL, ok := encoding.DecodeURL(p.config.HMACKey, vars["sigHash"], vars["encodedURL"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	gologit.Debugln("URL:", sURL)
	gologit.Debugln("Client request:", req)

	u, err := url.Parse(sURL)
	if err != nil {
		gologit.Debugln("url parse error:", err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	u.Host = strings.ToLower(u.Host)
	if u.Host == "" || localhostRegex.MatchString(u.Host) {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out rfc1918 hosts
	ip := net.ParseIP(u.Host)
	if ip != nil {
		if addr1918PrefixRegex.MatchString(ip.String()) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest(req.Method, sURL, nil)
	if err != nil {
		gologit.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	if req.Header.Get("X-Forwarded-For") == "" {
		host, _, err := net.SplitHostPort(req.RemoteAddr)
		if err == nil && !addr1918PrefixRegex.MatchString(host) {
			nreq.Header.Add("X-Forwarded-For", host)
		}
	}

	// add an accept header if the client didn't send one
	if nreq.Header.Get("Accept") == "" {
		nreq.Header.Add("Accept", "image/*")
	}

	nreq.Header.Add("user-agent", p.config.ServerName)
	nreq.Header.Add("via", p.config.ServerName)

	gologit.Debugln("Built outgoing request:", nreq)

	resp, err := p.client.Do(nreq)
	if err != nil {
		gologit.Debugln("Could not connect to endpoint", err)
		// this is a bit janky, but better than peeling off the
		// 3 layers of wrapped errors and trying to get to net.OpErr and
		// still having to rely on string comparison to find out if it is
		// a net.errClosing or not.
		errString := err.Error()
		if strings.Contains(errString, "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else if strings.Contains(errString, "use of closed") {
			http.Error(w, "Error Fetching Resource", http.StatusGatewayTimeout)
		} else {
			// some other error. call it a bad gateway
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		}
		return
	}
	defer resp.Body.Close()
	gologit.Debugln("Response from upstream:", resp)

	// check for too large a response
	if resp.ContentLength > p.config.MaxSize {
		gologit.Debugln("Content length exceeded", sURL)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		if !strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
			gologit.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		gologit.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303, 307:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	w.WriteHeader(resp.StatusCode)

	// since this uses io.Copy from the respBody, it is streaming
	// from the request to the response. This means it will nearly
	// always end up with a chunked response.
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		opErr, ok := err.(*net.OpError)
		if ok {
			switch opErr.Err {
			case syscall.EPIPE:
				// broken pipe - endpoint terminated the conn
				// log as debug only.
				gologit.Debugln("OpError writing response:", err)
			case syscall.ECONNRESET:
				// connection reset by peer - endpoint terminated the conn
				// log as debug only.
				gologit.Debugln("OpError writing response:", err)
			default:
				// log anything else normally
				gologit.Println("OpError writing response:", err)
			}
		} else {
			// unknown error and not an OpError.
			gologit.Println("Error writing response:", err)
		}
		return
	}

	if p.metrics != nil {
		go p.metrics.AddBytes(bW)
	}
	gologit.Debugln("Response to client:", w)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *Proxy) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// SetMetricsCollector sets a proxy metrics (ProxyMetrics interface) for
// the proxy
func (p *Proxy) SetMetricsCollector(pm ProxyMetrics) {
	p.metrics = pm
}

// New returns a new Proxy. An error is returned if there was a failure
// to parse the regex from the passed Config.
func New(pc Config) (*Proxy, error) {
	tr := &httpclient.Transport{
		MaxIdleConnsPerHost: 8,
		ConnectTimeout:      2 * time.Second,
		RequestTimeout:      pc.RequestTimeout}

	// spawn an idle conn trimmer
	go func() {
		// prunes every 5 minutes. this is just a guess at an
		// initial value. very busy severs may want to lower this...
		for {
			time.Sleep(5 * time.Minute)
			tr.CloseIdleConnections()
		}
	}()

	client := &http.Client{Transport: tr}
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= pc.MaxRedirects {
			return errors.New("Too many redirects")
		}
		return nil
	}

	var allow []*regexp.Regexp
	var c *regexp.Regexp
	var err error
	// compile allow list
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &Proxy{
		client:    client,
		config:    &pc,
		allowList: allow}, nil
}
<|endoftext|>


<commit_before>package canal

import (
	"bytes"
	"fmt"

	"github.com/siddontang/go-mysql/schema"
)

const (
	UpdateAction = "update"
	InsertAction = "insert"
	DeleteAction = "delete"
)

type RowsEvent struct {
	Table  *schema.Table
	Action string
	// changed row list
	// binlog has three update event version, v0, v1 and v2.
	// for v1 and v2, the rows number must be even.
	// Two rows for one event, format is [before update row, after update row]
	// for update v0, only one row for a event, and we don't support this version.
	Rows [][]interface{}
}

func newRowsEvent(table *schema.Table, action string, rows [][]interface{}) *RowsEvent {
	e := new(RowsEvent)

	e.Table = table
	e.Action = action
	e.Rows = rows

	return e
}

// Get primary keys in one row for a table, a table may use multi fields as the PK
func GetPKValues(table *schema.Table, row []interface{}) ([]interface{}, error) {
	indexes := table.PKColumns
	if len(indexes) == 0 {
		return nil, fmt.Errorf("table %s has no PK", table)
	} else if len(table.Columns) >= len(row) {
		return nil, fmt.Errorf("table %s has %d columns, but row data %v len is %d", table,
			len(table.Columns), row, len(row))
	}

	values := make([]interface{}, 0, len(indexes))

	for _, index := range indexes {
		values = append(values, row[index])
	}

	return values, nil
}

func FormatPKValues(pks []interface{}) string {
	var buf bytes.Buffer

	sep := ""
	for _, value := range pks {
		buf.WriteString(fmt.Sprintf("%s%v", sep, value))
		sep = ":"
	}

	return buf.String()
}
<commit_msg>update canal<commit_after>package canal

import (
	"fmt"

	"github.com/siddontang/go-mysql/schema"
)

const (
	UpdateAction = "update"
	InsertAction = "insert"
	DeleteAction = "delete"
)

type RowsEvent struct {
	Table  *schema.Table
	Action string
	// changed row list
	// binlog has three update event version, v0, v1 and v2.
	// for v1 and v2, the rows number must be even.
	// Two rows for one event, format is [before update row, after update row]
	// for update v0, only one row for a event, and we don't support this version.
	Rows [][]interface{}
}

func newRowsEvent(table *schema.Table, action string, rows [][]interface{}) *RowsEvent {
	e := new(RowsEvent)

	e.Table = table
	e.Action = action
	e.Rows = rows

	return e
}

// Get primary keys in one row for a table, a table may use multi fields as the PK
func GetPKValues(table *schema.Table, row []interface{}) ([]interface{}, error) {
	indexes := table.PKColumns
	if len(indexes) == 0 {
		return nil, fmt.Errorf("table %s has no PK", table)
	} else if len(table.Columns) >= len(row) {
		return nil, fmt.Errorf("table %s has %d columns, but row data %v len is %d", table,
			len(table.Columns), row, len(row))
	}

	values := make([]interface{}, 0, len(indexes))

	for _, index := range indexes {
		values = append(values, row[index])
	}

	return values, nil
}
<|endoftext|>


<commit_before>package ircbot

import (
	"fmt"
	"math/rand"
	"strings"
)

type ActionFunc func(b *IrcBot, m *IrcMsg)

type Action struct {
	command string
	do      ActionFunc
}

func Pong(b *IrcBot, m *IrcMsg) {

	s := fmt.Sprintf("%s", strings.Join(m.args, " "))
	b.Out <- &IrcMsg{
		command: "PONG",
		args:    []string{s},
	}
}

func Join(b *IrcBot, m *IrcMsg) {
	if m.nick == b.Nick {
		b.joined = true
		return
	}

	s := fmt.Sprintf("%s :Salut %s", b.Channel[0], m.nick)
	b.Out <- &IrcMsg{
		command: "PRIVMSG",
		args:    []string{s},
	}
}

func Respond(b *IrcBot, m *IrcMsg) {
	response := []string{
		"oui ?",
		"on parle de moi ?",
		"Je suis pas là",
	}

	s := strings.Join(m.args, " ")

	if strings.Contains(s, b.Nick) {
		nbr := rand.Intn(len(response))
		line := fmt.Sprintf("%s :%s", b.Channel[0], response[nbr])
		b.Out <- &IrcMsg{
			command: "PRIVMSG",
			args:    []string{line},
		}
	}
}
<commit_msg>always respond to ping<commit_after>package ircbot

import (
	"fmt"
	"math/rand"
	"strings"
)

type ActionFunc func(b *IrcBot, m *IrcMsg)

type Action struct {
	command string
	do      ActionFunc
}

func Pong(b *IrcBot, m *IrcMsg) {

	s := fmt.Sprintf("PONG %s", strings.Join(m.args, " "))
	fmt.Println("irc >> ", s)
	b.writer.PrintfLine(s)
}

func Join(b *IrcBot, m *IrcMsg) {
	if m.nick == b.Nick {
		b.joined = true
		return
	}

	s := fmt.Sprintf("%s :Salut %s", b.Channel[0], m.nick)
	b.Out <- &IrcMsg{
		command: "PRIVMSG",
		args:    []string{s},
	}
}

func Respond(b *IrcBot, m *IrcMsg) {
	response := []string{
		"oui ?",
		"on parle de moi ?",
		"Je suis pas là",
	}

	s := strings.Join(m.args, " ")

	if strings.Contains(s, b.Nick) {
		nbr := rand.Intn(len(response))
		line := fmt.Sprintf("%s :%s", b.Channel[0], response[nbr])
		b.Out <- &IrcMsg{
			command: "PRIVMSG",
			args:    []string{line},
		}
	}
}
<|endoftext|>


<commit_before>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filer           *Filer
	self            pb.ServerAddress
	isLeader        bool
	grpcDialOption  grpc.DialOption
	MetaLogBuffer   *log_buffer.LogBuffer
	peerStatues     map[pb.ServerAddress]int
	peerStatuesLock sync.Mutex
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filer:          filer,
		self:           self,
		grpcDialOption: grpcDialOption,
		peerStatues:    make(map[pb.ServerAddress]int),
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
	if update.NodeType != cluster.FilerType {
		return
	}

	address := pb.ServerAddress(update.Address)
	if update.IsAdd {
		// every filer should subscribe to a new filer
		if ma.setActive(address, true) {
			go ma.loopSubscribeToOnefiler(ma.filer, ma.self, address, startFrom)
		}
	} else {
		ma.setActive(address, false)
	}
}

func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	if isActive {
		if _, found := ma.peerStatues[address]; found {
			ma.peerStatues[address] += 1
		} else {
			ma.peerStatues[address] = 1
			notDuplicated = true
		}
	} else {
		if _, found := ma.peerStatues[address]; found {
			delete(ma.peerStatues, address)
		}
	}
	return
}
func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	var count int
	count, isActive = ma.peerStatues[address]
	return count > 0 && isActive
}

func (ma *MetaAggregator) loopSubscribeToOnefiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time) {
	lastTsNs := startFrom.UnixNano()
	for {
		glog.V(0).Infof("loopSubscribeToOnefiler read %s start from %v %d", peer, time.Unix(0, lastTsNs), lastTsNs)
		nextLastTsNs, err := ma.doSubscribeToOneFiler(f, self, peer, lastTsNs)
		if !ma.isActive(peer) {
			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
			return
		}
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
		} else if lastTsNs < nextLastTsNs {
			lastTsNs = nextLastTsNs
		}
		time.Sleep(1733 * time.Millisecond)
	}
}

func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom int64) (int64, error) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := startFrom

	peerSignature, err := ma.readFilerStoreSignature(peer)
	if err != nil {
		return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err)
	}

	// when filer store is not shared by multiple filers
	if peerSignature != f.Signature {
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
			defer func(prevTsNs int64) {
				if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
					if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
						glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
					} else {
						glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
					}
				}
			}(prevTsNs)
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		var counter int64
		var synced bool
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			counter++
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
					} else if !synced {
						synced = true
						glog.V(0).Infof("synced with %s", peer)
					}
					lastPersistTime = time.Now()
					counter = 0
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
	err = pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
			ClientName: "filer:" + string(self),
			PathPrefix: "/",
			SinceNs:    lastTsNs,
			ClientId:   int32(ma.filer.UniqueFileId),
		})
		if err != nil {
			return fmt.Errorf("subscribe: %v", err)
		}

		for {
			resp, listenErr := stream.Recv()
			if listenErr == io.EOF {
				return nil
			}
			if listenErr != nil {
				return listenErr
			}

			if err := processEventFn(resp); err != nil {
				return fmt.Errorf("process %v: %v", resp, err)
			}
			lastTsNs = resp.TsNs

			f.onMetadataChangeEvent(resp)

		}
	})
	return lastTsNs, err
}

func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
	err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const (
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<commit_msg>Avoid synchronizing from scratch due to err<commit_after>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filer           *Filer
	self            pb.ServerAddress
	isLeader        bool
	grpcDialOption  grpc.DialOption
	MetaLogBuffer   *log_buffer.LogBuffer
	peerStatues     map[pb.ServerAddress]int
	peerStatuesLock sync.Mutex
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filer:          filer,
		self:           self,
		grpcDialOption: grpcDialOption,
		peerStatues:    make(map[pb.ServerAddress]int),
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
	if update.NodeType != cluster.FilerType {
		return
	}

	address := pb.ServerAddress(update.Address)
	if update.IsAdd {
		// every filer should subscribe to a new filer
		if ma.setActive(address, true) {
			go ma.loopSubscribeToOnefiler(ma.filer, ma.self, address, startFrom)
		}
	} else {
		ma.setActive(address, false)
	}
}

func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	if isActive {
		if _, found := ma.peerStatues[address]; found {
			ma.peerStatues[address] += 1
		} else {
			ma.peerStatues[address] = 1
			notDuplicated = true
		}
	} else {
		if _, found := ma.peerStatues[address]; found {
			delete(ma.peerStatues, address)
		}
	}
	return
}
func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	var count int
	count, isActive = ma.peerStatues[address]
	return count > 0 && isActive
}

func (ma *MetaAggregator) loopSubscribeToOnefiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time) {
	lastTsNs := startFrom.UnixNano()
	for {
		glog.V(0).Infof("loopSubscribeToOnefiler read %s start from %v %d", peer, time.Unix(0, lastTsNs), lastTsNs)
		nextLastTsNs, err := ma.doSubscribeToOneFiler(f, self, peer, lastTsNs)
		if !ma.isActive(peer) {
			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
			return
		}
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
		}
		if lastTsNs < nextLastTsNs {
			lastTsNs = nextLastTsNs
		}
		time.Sleep(1733 * time.Millisecond)
	}
}

func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom int64) (int64, error) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := startFrom

	peerSignature, err := ma.readFilerStoreSignature(peer)
	if err != nil {
		return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err)
	}

	// when filer store is not shared by multiple filers
	if peerSignature != f.Signature {
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
			defer func(prevTsNs int64) {
				if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
					if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
						glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
					} else {
						glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
					}
				}
			}(prevTsNs)
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		var counter int64
		var synced bool
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			counter++
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
					} else if !synced {
						synced = true
						glog.V(0).Infof("synced with %s", peer)
					}
					lastPersistTime = time.Now()
					counter = 0
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
	err = pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
			ClientName: "filer:" + string(self),
			PathPrefix: "/",
			SinceNs:    lastTsNs,
			ClientId:   int32(ma.filer.UniqueFileId),
		})
		if err != nil {
			return fmt.Errorf("subscribe: %v", err)
		}

		for {
			resp, listenErr := stream.Recv()
			if listenErr == io.EOF {
				return nil
			}
			if listenErr != nil {
				return listenErr
			}

			if err := processEventFn(resp); err != nil {
				return fmt.Errorf("process %v: %v", resp, err)
			}
			lastTsNs = resp.TsNs

			f.onMetadataChangeEvent(resp)

		}
	})
	return lastTsNs, err
}

func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
	err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const (
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<|endoftext|>


<commit_before>package service

import (
	"cf"
	"cf/api"
	"cf/configuration"
	"cf/errors"
	"cf/models"
	"cf/requirements"
	"cf/terminal"
	"github.com/codegangsta/cli"
)

type BindService struct {
	ui                 terminal.UI
	config             configuration.Reader
	serviceBindingRepo api.ServiceBindingRepository
	appReq             requirements.ApplicationRequirement
	serviceInstanceReq requirements.ServiceInstanceRequirement
}

type ServiceBinder interface {
	BindApplication(app models.Application, serviceInstance models.ServiceInstance) (apiErr error)
}

func NewBindService(ui terminal.UI, config configuration.Reader, serviceBindingRepo api.ServiceBindingRepository) (cmd *BindService) {
	cmd = new(BindService)
	cmd.ui = ui
	cmd.config = config
	cmd.serviceBindingRepo = serviceBindingRepo
	return
}

func (cmd *BindService) GetRequirements(reqFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {

	if len(c.Args()) != 2 {
		err = errors.New("Incorrect Usage")
		cmd.ui.FailWithUsage(c, "bind-service")
		return
	}
	appName := c.Args()[0]
	serviceName := c.Args()[1]

	cmd.appReq = reqFactory.NewApplicationRequirement(appName)
	cmd.serviceInstanceReq = reqFactory.NewServiceInstanceRequirement(serviceName)

	reqs = []requirements.Requirement{cmd.appReq, cmd.serviceInstanceReq}
	return
}

func (cmd *BindService) Run(c *cli.Context) {
	app := cmd.appReq.GetApplication()
	serviceInstance := cmd.serviceInstanceReq.GetServiceInstance()

	cmd.ui.Say("Binding service %s to app %s in org %s / space %s as %s...",
		terminal.EntityNameColor(serviceInstance.Name),
		terminal.EntityNameColor(app.Name),
		terminal.EntityNameColor(cmd.config.OrganizationFields().Name),
		terminal.EntityNameColor(cmd.config.SpaceFields().Name),
		terminal.EntityNameColor(cmd.config.Username()),
	)

	err := cmd.BindApplication(app, serviceInstance)
	if err != nil {
		if err, ok := err.(errors.HttpError); ok && err.ErrorCode() == errors.APP_ALREADY_BOUND {
			cmd.ui.Ok()
			cmd.ui.Warn("App %s is already bound to %s.", app.Name, serviceInstance.Name)
			return
		} else {
			cmd.ui.Failed(err.Error())
		}
	}

	cmd.ui.Ok()
	cmd.ui.Say("TIP: Use '%s push' to ensure your env variable changes take effect", cf.Name())
}

func (cmd *BindService) BindApplication(app models.Application, serviceInstance models.ServiceInstance) (apiErr error) {
	apiErr = cmd.serviceBindingRepo.Create(serviceInstance.Guid, app.Guid)
	return
}
<commit_msg>Color code TIP message in "cf bind-service"<commit_after>package service

import (
	"cf"
	"cf/api"
	"cf/configuration"
	"cf/errors"
	"cf/models"
	"cf/requirements"
	"cf/terminal"
	"github.com/codegangsta/cli"
)

type BindService struct {
	ui                 terminal.UI
	config             configuration.Reader
	serviceBindingRepo api.ServiceBindingRepository
	appReq             requirements.ApplicationRequirement
	serviceInstanceReq requirements.ServiceInstanceRequirement
}

type ServiceBinder interface {
	BindApplication(app models.Application, serviceInstance models.ServiceInstance) (apiErr error)
}

func NewBindService(ui terminal.UI, config configuration.Reader, serviceBindingRepo api.ServiceBindingRepository) (cmd *BindService) {
	cmd = new(BindService)
	cmd.ui = ui
	cmd.config = config
	cmd.serviceBindingRepo = serviceBindingRepo
	return
}

func (cmd *BindService) GetRequirements(reqFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {

	if len(c.Args()) != 2 {
		err = errors.New("Incorrect Usage")
		cmd.ui.FailWithUsage(c, "bind-service")
		return
	}
	appName := c.Args()[0]
	serviceName := c.Args()[1]

	cmd.appReq = reqFactory.NewApplicationRequirement(appName)
	cmd.serviceInstanceReq = reqFactory.NewServiceInstanceRequirement(serviceName)

	reqs = []requirements.Requirement{cmd.appReq, cmd.serviceInstanceReq}
	return
}

func (cmd *BindService) Run(c *cli.Context) {
	app := cmd.appReq.GetApplication()
	serviceInstance := cmd.serviceInstanceReq.GetServiceInstance()

	cmd.ui.Say("Binding service %s to app %s in org %s / space %s as %s...",
		terminal.EntityNameColor(serviceInstance.Name),
		terminal.EntityNameColor(app.Name),
		terminal.EntityNameColor(cmd.config.OrganizationFields().Name),
		terminal.EntityNameColor(cmd.config.SpaceFields().Name),
		terminal.EntityNameColor(cmd.config.Username()),
	)

	err := cmd.BindApplication(app, serviceInstance)
	if err != nil {
		if err, ok := err.(errors.HttpError); ok && err.ErrorCode() == errors.APP_ALREADY_BOUND {
			cmd.ui.Ok()
			cmd.ui.Warn("App %s is already bound to %s.", app.Name, serviceInstance.Name)
			return
		} else {
			cmd.ui.Failed(err.Error())
		}
	}

	cmd.ui.Ok()
	cmd.ui.Say("TIP: Use '%s' to ensure your env variable changes take effect", terminal.CommandColor(cf.Name()+" push"))
}

func (cmd *BindService) BindApplication(app models.Application, serviceInstance models.ServiceInstance) (apiErr error) {
	apiErr = cmd.serviceBindingRepo.Create(serviceInstance.Guid, app.Guid)
	return
}
<|endoftext|>


<commit_before>package main

import (
	mwclient "cgt.name/pkg/go-mwclient"
	"cgt.name/pkg/go-mwclient/params"
	"fmt"
	"github.com/antonholmquist/jason"
	"github.com/garyhouston/takenwith/mwlib"
)

func requestCategories(page string, client *mwclient.Client) *jason.Object {
	params := params.Values{
		"action":  "query",
		"titles":  page,
		"prop":    "categories",
		"cllimit": "max",
	}
	json, err := client.Get(params)
	if err != nil {
		panic(err)
	}
	return json
}

func getPageCategoriesOld(page string, client *mwclient.Client) []string {
	json := requestCategories(page, client)
	pageObj := mwlib.GetJsonPage(json)
	if pageObj == nil {
		panic(fmt.Sprintf("page %v not found", page))
	}
	catJson, err := pageObj.GetObjectArray("categories")
	if err != nil {
		// Image has no categories? may be able to return empty
		// array.
		panic(fmt.Sprintf("No categories? json: %v ", json))
	}
	var categories = make([]string, 0, 20)
	for i := range catJson {
		title, err := catJson[i].GetString("title")
		if err != nil {
			panic(err)
		}
		categories = append(categories, title)
	}
	return categories
}

// Given an array of page titles, return a mapping from page title to the array
// of categories which the page is a member of.
// If the page doesn't exist, no entry is added to the map.
// If the page has no categories, it will map to nil.
func getPageCategories(pages []string, client *mwclient.Client) map[string][]string {
	params := params.Values{
		"action":   "query",
		"titles":   mwlib.MakeTitleString(pages),
		"prop":     "categories",
		"cllimit":  "max",
		"continue": "",
	}
	json, err := client.Post(params) // Get may fail on long queries.
	if err != nil {
		fmt.Println(params)
		panic(err)
	}
	pageData, err := json.GetObject("query", "pages")
	if err != nil {
		panic(err)
	}
	pageMap := pageData.Map()
	result := make(map[string][]string)
	for pageId, page := range pageMap {
		pageObj, err := page.Object()
		if err != nil {
			panic(err)
		}
		title, err := pageObj.GetString("title")
		if err != nil {
			panic(err)
		}
		if pageId[0] == '-' {
			fmt.Println(title)
			fmt.Println("File does not exist, possibly deleted.")
			continue
		}
		categories, err := pageObj.GetObjectArray("categories")
		if err != nil {
			// Presumably the page has no categories.
			result[title] = nil
			continue
		}
		catArray := make([]string, len(categories))
		for i := range categories {
			catArray[i], err = categories[i].GetString("title")
			if err != nil {
				panic(err)
			}
		}
		result[title] = catArray
	}
	return result
}
<commit_msg>Delete an obsolete func<commit_after>package main

import (
	mwclient "cgt.name/pkg/go-mwclient"
	"cgt.name/pkg/go-mwclient/params"
	"fmt"
	"github.com/antonholmquist/jason"
	"github.com/garyhouston/takenwith/mwlib"
)

func requestCategories(page string, client *mwclient.Client) *jason.Object {
	params := params.Values{
		"action":  "query",
		"titles":  page,
		"prop":    "categories",
		"cllimit": "max",
	}
	json, err := client.Get(params)
	if err != nil {
		panic(err)
	}
	return json
}

// Given an array of page titles, return a mapping from page title to the array
// of categories which the page is a member of.
// If the page doesn't exist, no entry is added to the map.
// If the page has no categories, it will map to nil.
func getPageCategories(pages []string, client *mwclient.Client) map[string][]string {
	params := params.Values{
		"action":   "query",
		"titles":   mwlib.MakeTitleString(pages),
		"prop":     "categories",
		"cllimit":  "max",
		"continue": "",
	}
	json, err := client.Post(params) // Get may fail on long queries.
	if err != nil {
		fmt.Println(params)
		panic(err)
	}
	pageData, err := json.GetObject("query", "pages")
	if err != nil {
		panic(err)
	}
	pageMap := pageData.Map()
	result := make(map[string][]string)
	for pageId, page := range pageMap {
		pageObj, err := page.Object()
		if err != nil {
			panic(err)
		}
		title, err := pageObj.GetString("title")
		if err != nil {
			panic(err)
		}
		if pageId[0] == '-' {
			fmt.Println(title)
			fmt.Println("File does not exist, possibly deleted.")
			continue
		}
		categories, err := pageObj.GetObjectArray("categories")
		if err != nil {
			// Presumably the page has no categories.
			result[title] = nil
			continue
		}
		catArray := make([]string, len(categories))
		for i := range categories {
			catArray[i], err = categories[i].GetString("title")
			if err != nil {
				panic(err)
			}
		}
		result[title] = catArray
	}
	return result
}
<|endoftext|>


<commit_before>package endpoints

import (
	"bytes"
	"context"
	"sort"

	"encoding/json"
	"fmt"
	"reflect"

	"net"

	workloadUtil "github.com/rancher/rancher/pkg/controllers/user/workload"
	nodehelper "github.com/rancher/rancher/pkg/node"
	"github.com/rancher/types/apis/core/v1"
	managementv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/apis/project.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/sets"
)

const (
	allEndpoints        = "_all_endpoints_"
	endpointsAnnotation = "field.cattle.io/publicEndpoints"
)

func Register(ctx context.Context, workload *config.UserContext) {
	isRKE := false
	cluster, err := workload.Management.Management.Clusters("").Get(workload.ClusterName, metav1.GetOptions{})
	if err != nil {
		logrus.WithError(err).Warnf("Can not get cluster %s when registering endpoint controller", workload.ClusterName)
	}
	if cluster != nil {
		//assume that cluster always has a spec
		isRKE = cluster.Spec.RancherKubernetesEngineConfig != nil
	}
	n := &NodesController{
		nodes:              workload.Core.Nodes(""),
		serviceLister:      workload.Core.Services("").Controller().Lister(),
		nodeLister:         workload.Core.Nodes("").Controller().Lister(),
		podLister:          workload.Core.Pods("").Controller().Lister(),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		clusterName:        workload.ClusterName,
	}
	workload.Core.Nodes("").AddHandler("nodesEndpointsController", n.sync)

	s := &ServicesController{
		services:           workload.Core.Services(""),
		serviceLister:      workload.Core.Services("").Controller().Lister(),
		nodeLister:         workload.Core.Nodes("").Controller().Lister(),
		nodeController:     workload.Core.Nodes("").Controller(),
		podLister:          workload.Core.Pods("").Controller().Lister(),
		podController:      workload.Core.Pods("").Controller(),
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		clusterName:        workload.ClusterName,
	}
	workload.Core.Services("").AddHandler("servicesEndpointsController", s.sync)

	p := &PodsController{
		nodeLister:         workload.Core.Nodes("").Controller().Lister(),
		nodeController:     workload.Core.Nodes("").Controller(),
		pods:               workload.Core.Pods(""),
		serviceLister:      workload.Core.Services("").Controller().Lister(),
		podLister:          workload.Core.Pods("").Controller().Lister(),
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		clusterName:        workload.ClusterName,
	}
	workload.Core.Pods("").AddHandler("hostPortEndpointsController", p.sync)

	w := &WorkloadEndpointsController{
		ingressLister:  workload.Extensions.Ingresses("").Controller().Lister(),
		serviceLister:  workload.Core.Services("").Controller().Lister(),
		podLister:      workload.Core.Pods("").Controller().Lister(),
		machinesLister: workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		nodeLister:     workload.Core.Nodes("").Controller().Lister(),
		clusterName:    workload.ClusterName,
		isRKE:          isRKE,
	}
	w.WorkloadController = workloadUtil.NewWorkloadController(workload.UserOnlyContext(), w.UpdateEndpoints)

	i := &IngressEndpointsController{
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		ingressInterface:   workload.Extensions.Ingresses(""),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		isRKE:              isRKE,
		clusterName:        workload.ClusterName,
	}
	workload.Extensions.Ingresses("").AddHandler("ingressEndpointsController", i.sync)
}

func areEqualEndpoints(one []v3.PublicEndpoint, two []v3.PublicEndpoint) bool {
	oneMap := map[string]bool{}
	twoMap := map[string]bool{}
	for _, value := range one {
		oneMap[publicEndpointToString(value)] = true
	}
	for _, value := range two {
		twoMap[publicEndpointToString(value)] = true
	}
	return reflect.DeepEqual(oneMap, twoMap)
}

func publicEndpointsToString(eps []v3.PublicEndpoint) (string, error) {
	b, err := json.Marshal(eps)
	if err != nil {
		return "", err
	}
	return string(b), nil
}

func getPublicEndpointsFromAnnotations(annotations map[string]string) []v3.PublicEndpoint {
	var eps []v3.PublicEndpoint
	if annotations == nil {
		return eps
	}
	if val, ok := annotations[endpointsAnnotation]; ok {
		err := json.Unmarshal([]byte(val), &eps)
		if err != nil {
			logrus.Errorf("Failed to read public endpoints from annotation %v", err)
			return eps
		}
	}
	return eps
}

func convertServiceToPublicEndpoints(svc *corev1.Service, clusterName string, node *managementv3.Node, allNodesIP string) ([]v3.PublicEndpoint, error) {
	var eps []v3.PublicEndpoint
	if svc.DeletionTimestamp != nil {
		return eps, nil
	}
	if !(svc.Spec.Type == "NodePort" || svc.Spec.Type == "LoadBalancer") {
		return eps, nil
	}
	address := ""
	nodeName := ""
	if node != nil {
		address = nodehelper.GetEndpointNodeIP(node)
		nodeName = fmt.Sprintf("%s:%s", clusterName, node.Name)
	}

	svcName := fmt.Sprintf("%s:%s", svc.Namespace, svc.Name)
	if svc.Spec.Type == "NodePort" {
		addresses := []string{}
		if allNodesIP != "" {
			addresses = append(addresses, allNodesIP)
		}
		for _, port := range svc.Spec.Ports {
			if port.NodePort == 0 {
				continue
			}
			p := v3.PublicEndpoint{
				NodeName:    nodeName,
				Port:        port.NodePort,
				Addresses:   addresses,
				Protocol:    string(port.Protocol),
				ServiceName: svcName,
				AllNodes:    true,
			}
			//for getting endpoints of specific node
			if address != "" {
				p.Addresses = append(p.Addresses, address)
			}
			eps = append(eps, p)
		}
	} else {
		var addresses []string
		for _, ingressEp := range svc.Status.LoadBalancer.Ingress {
			address := ingressEp.Hostname
			if address == "" {
				address = ingressEp.IP
			}
			addresses = append(addresses, address)
		}
		if len(addresses) > 0 {
			for _, port := range svc.Spec.Ports {
				p := v3.PublicEndpoint{
					NodeName:    "",
					Addresses:   addresses,
					Port:        port.Port,
					Protocol:    string(port.Protocol),
					ServiceName: svcName,
					AllNodes:    false,
				}
				eps = append(eps, p)
			}
		}
	}

	return eps, nil
}

func convertHostPortToEndpoint(pod *corev1.Pod, clusterName string, node *managementv3.Node) ([]v3.PublicEndpoint, error) {
	var eps []v3.PublicEndpoint
	if pod.DeletionTimestamp != nil {
		return eps, nil
	}
	if pod.Status.Phase != corev1.PodRunning {
		return eps, nil
	}
	if node == nil {
		return eps, nil
	}
	for _, c := range pod.Spec.Containers {
		for _, p := range c.Ports {
			if p.HostPort == 0 {
				continue
			}
			address := nodehelper.GetEndpointNodeIP(node)
			p := v3.PublicEndpoint{
				NodeName:  fmt.Sprintf("%s:%s", clusterName, node.Name),
				Addresses: []string{address},
				Port:      p.HostPort,
				Protocol:  string(p.Protocol),
				PodName:   fmt.Sprintf("%s:%s", pod.Namespace, pod.Name),
				AllNodes:  false,
			}
			eps = append(eps, p)
		}
	}

	return eps, nil
}

func publicEndpointToString(p v3.PublicEndpoint) string {
	sort.Strings(p.Addresses)
	return fmt.Sprintf("%s_%v_%v_%s_%s_%s_%s_%s_%s", p.NodeName, p.Addresses, p.Port, p.Protocol, p.ServiceName, p.PodName, p.IngressName, p.Hostname, p.Path)
}

func getNodeNameToMachine(clusterName string, machineLister managementv3.NodeLister, nodeLister v1.NodeLister) (map[string]*managementv3.Node, error) {
	machines, err := machineLister.List(clusterName, labels.NewSelector())
	if err != nil {
		return nil, err
	}
	machineMap := map[string]*managementv3.Node{}
	nodes, err := nodeLister.List("", labels.NewSelector())
	if err != nil {
		return nil, err
	}
	for _, machine := range machines {
		for _, node := range nodes {
			if nodehelper.IsNodeForNode(node, machine) {
				machineMap[node.Name] = machine
			}
		}
	}
	return machineMap, nil
}

func isMachineReady(machine *managementv3.Node) bool {
	for _, cond := range machine.Status.InternalNodeStatus.Conditions {
		if cond.Type == corev1.NodeReady {
			return cond.Status == corev1.ConditionTrue
		}
	}
	return false
}

func getAllNodesPublicEndpointIP(machineLister managementv3.NodeLister, clusterName string) (string, error) {
	var addresses []net.IP
	machines, err := machineLister.List(clusterName, labels.NewSelector())
	if err != nil {
		return "", err
	}
	for _, machine := range machines {
		if machine.Spec.Worker && isMachineReady(machine) {
			nodePublicIP := net.ParseIP(nodehelper.GetEndpointNodeIP(machine))
			if nodePublicIP.String() != "" {
				addresses = append(addresses, nodePublicIP)
			}
		}
	}
	if len(addresses) == 0 {
		return "", nil
	}

	sort.Slice(addresses, func(i, j int) bool {
		return bytes.Compare(addresses[i], addresses[j]) < 0
	})

	return addresses[0].String(), nil
}

func convertIngressToServicePublicEndpointsMap(obj *extensionsv1beta1.Ingress, allNodes bool, allNodesIP string) map[string][]v3.PublicEndpoint {
	var addresses []string
	epsMap := map[string][]v3.PublicEndpoint{}
	if !allNodes {
		for _, address := range obj.Status.LoadBalancer.Ingress {
			addresses = append(addresses, address.IP)
		}

	} else {
		if allNodesIP != "" {
			addresses = append(addresses, allNodesIP)
		}
	}

	if len(addresses) == 0 {
		return epsMap
	}

	tlsHosts := sets.NewString()
	for _, t := range obj.Spec.TLS {
		tlsHosts.Insert(t.Hosts...)
	}

	ports := map[int32]string{80: "HTTP", 443: "HTTPS"}
	for _, rule := range obj.Spec.Rules {
		for _, path := range rule.HTTP.Paths {
			for port, proto := range ports {
				if port == 80 {
					if tlsHosts.Has(rule.Host) {
						continue
					}
				} else {
					if !tlsHosts.Has(rule.Host) && rule.Host != "" {
						continue
					}
				}
				p := v3.PublicEndpoint{
					Hostname:    rule.Host,
					Path:        path.Path,
					ServiceName: fmt.Sprintf("%s:%s", obj.Namespace, path.Backend.ServiceName),
					Addresses:   addresses,
					Port:        port,
					Protocol:    proto,
					AllNodes:    allNodes,
					IngressName: fmt.Sprintf("%s:%s", obj.Namespace, obj.Name),
				}
				epsMap[path.Backend.ServiceName] = append(epsMap[path.Backend.ServiceName], p)
			}
		}
	}
	return epsMap
}

func convertIngressToPublicEndpoints(obj *extensionsv1beta1.Ingress, isRKE bool, allNodesIP string) []v3.PublicEndpoint {
	epsMap := convertIngressToServicePublicEndpointsMap(obj, isRKE, allNodesIP)
	var eps []v3.PublicEndpoint
	for _, v := range epsMap {
		eps = append(eps, v...)
	}
	return eps
}
<commit_msg>Show Ingress Endpoint after generated<commit_after>package endpoints

import (
	"bytes"
	"context"
	"sort"

	"github.com/rancher/rancher/pkg/settings"

	"encoding/json"
	"fmt"
	"reflect"

	"net"

	workloadUtil "github.com/rancher/rancher/pkg/controllers/user/workload"
	nodehelper "github.com/rancher/rancher/pkg/node"
	"github.com/rancher/types/apis/core/v1"
	managementv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/apis/project.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/sets"
)

const (
	allEndpoints        = "_all_endpoints_"
	endpointsAnnotation = "field.cattle.io/publicEndpoints"
)

func Register(ctx context.Context, workload *config.UserContext) {
	isRKE := false
	cluster, err := workload.Management.Management.Clusters("").Get(workload.ClusterName, metav1.GetOptions{})
	if err != nil {
		logrus.WithError(err).Warnf("Can not get cluster %s when registering endpoint controller", workload.ClusterName)
	}
	if cluster != nil {
		//assume that cluster always has a spec
		isRKE = cluster.Spec.RancherKubernetesEngineConfig != nil
	}
	n := &NodesController{
		nodes:              workload.Core.Nodes(""),
		serviceLister:      workload.Core.Services("").Controller().Lister(),
		nodeLister:         workload.Core.Nodes("").Controller().Lister(),
		podLister:          workload.Core.Pods("").Controller().Lister(),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		clusterName:        workload.ClusterName,
	}
	workload.Core.Nodes("").AddHandler("nodesEndpointsController", n.sync)

	s := &ServicesController{
		services:           workload.Core.Services(""),
		serviceLister:      workload.Core.Services("").Controller().Lister(),
		nodeLister:         workload.Core.Nodes("").Controller().Lister(),
		nodeController:     workload.Core.Nodes("").Controller(),
		podLister:          workload.Core.Pods("").Controller().Lister(),
		podController:      workload.Core.Pods("").Controller(),
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		clusterName:        workload.ClusterName,
	}
	workload.Core.Services("").AddHandler("servicesEndpointsController", s.sync)

	p := &PodsController{
		nodeLister:         workload.Core.Nodes("").Controller().Lister(),
		nodeController:     workload.Core.Nodes("").Controller(),
		pods:               workload.Core.Pods(""),
		serviceLister:      workload.Core.Services("").Controller().Lister(),
		podLister:          workload.Core.Pods("").Controller().Lister(),
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		clusterName:        workload.ClusterName,
	}
	workload.Core.Pods("").AddHandler("hostPortEndpointsController", p.sync)

	w := &WorkloadEndpointsController{
		ingressLister:  workload.Extensions.Ingresses("").Controller().Lister(),
		serviceLister:  workload.Core.Services("").Controller().Lister(),
		podLister:      workload.Core.Pods("").Controller().Lister(),
		machinesLister: workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		nodeLister:     workload.Core.Nodes("").Controller().Lister(),
		clusterName:    workload.ClusterName,
		isRKE:          isRKE,
	}
	w.WorkloadController = workloadUtil.NewWorkloadController(workload.UserOnlyContext(), w.UpdateEndpoints)

	i := &IngressEndpointsController{
		workloadController: workloadUtil.NewWorkloadController(workload.UserOnlyContext(), nil),
		ingressInterface:   workload.Extensions.Ingresses(""),
		machinesLister:     workload.Management.Management.Nodes(workload.ClusterName).Controller().Lister(),
		isRKE:              isRKE,
		clusterName:        workload.ClusterName,
	}
	workload.Extensions.Ingresses("").AddHandler("ingressEndpointsController", i.sync)
}

func areEqualEndpoints(one []v3.PublicEndpoint, two []v3.PublicEndpoint) bool {
	oneMap := map[string]bool{}
	twoMap := map[string]bool{}
	for _, value := range one {
		oneMap[publicEndpointToString(value)] = true
	}
	for _, value := range two {
		twoMap[publicEndpointToString(value)] = true
	}
	return reflect.DeepEqual(oneMap, twoMap)
}

func publicEndpointsToString(eps []v3.PublicEndpoint) (string, error) {
	b, err := json.Marshal(eps)
	if err != nil {
		return "", err
	}
	return string(b), nil
}

func getPublicEndpointsFromAnnotations(annotations map[string]string) []v3.PublicEndpoint {
	var eps []v3.PublicEndpoint
	if annotations == nil {
		return eps
	}
	if val, ok := annotations[endpointsAnnotation]; ok {
		err := json.Unmarshal([]byte(val), &eps)
		if err != nil {
			logrus.Errorf("Failed to read public endpoints from annotation %v", err)
			return eps
		}
	}
	return eps
}

func convertServiceToPublicEndpoints(svc *corev1.Service, clusterName string, node *managementv3.Node, allNodesIP string) ([]v3.PublicEndpoint, error) {
	var eps []v3.PublicEndpoint
	if svc.DeletionTimestamp != nil {
		return eps, nil
	}
	if !(svc.Spec.Type == "NodePort" || svc.Spec.Type == "LoadBalancer") {
		return eps, nil
	}
	address := ""
	nodeName := ""
	if node != nil {
		address = nodehelper.GetEndpointNodeIP(node)
		nodeName = fmt.Sprintf("%s:%s", clusterName, node.Name)
	}

	svcName := fmt.Sprintf("%s:%s", svc.Namespace, svc.Name)
	if svc.Spec.Type == "NodePort" {
		addresses := []string{}
		if allNodesIP != "" {
			addresses = append(addresses, allNodesIP)
		}
		for _, port := range svc.Spec.Ports {
			if port.NodePort == 0 {
				continue
			}
			p := v3.PublicEndpoint{
				NodeName:    nodeName,
				Port:        port.NodePort,
				Addresses:   addresses,
				Protocol:    string(port.Protocol),
				ServiceName: svcName,
				AllNodes:    true,
			}
			//for getting endpoints of specific node
			if address != "" {
				p.Addresses = append(p.Addresses, address)
			}
			eps = append(eps, p)
		}
	} else {
		var addresses []string
		for _, ingressEp := range svc.Status.LoadBalancer.Ingress {
			address := ingressEp.Hostname
			if address == "" {
				address = ingressEp.IP
			}
			addresses = append(addresses, address)
		}
		if len(addresses) > 0 {
			for _, port := range svc.Spec.Ports {
				p := v3.PublicEndpoint{
					NodeName:    "",
					Addresses:   addresses,
					Port:        port.Port,
					Protocol:    string(port.Protocol),
					ServiceName: svcName,
					AllNodes:    false,
				}
				eps = append(eps, p)
			}
		}
	}

	return eps, nil
}

func convertHostPortToEndpoint(pod *corev1.Pod, clusterName string, node *managementv3.Node) ([]v3.PublicEndpoint, error) {
	var eps []v3.PublicEndpoint
	if pod.DeletionTimestamp != nil {
		return eps, nil
	}
	if pod.Status.Phase != corev1.PodRunning {
		return eps, nil
	}
	if node == nil {
		return eps, nil
	}
	for _, c := range pod.Spec.Containers {
		for _, p := range c.Ports {
			if p.HostPort == 0 {
				continue
			}
			address := nodehelper.GetEndpointNodeIP(node)
			p := v3.PublicEndpoint{
				NodeName:  fmt.Sprintf("%s:%s", clusterName, node.Name),
				Addresses: []string{address},
				Port:      p.HostPort,
				Protocol:  string(p.Protocol),
				PodName:   fmt.Sprintf("%s:%s", pod.Namespace, pod.Name),
				AllNodes:  false,
			}
			eps = append(eps, p)
		}
	}

	return eps, nil
}

func publicEndpointToString(p v3.PublicEndpoint) string {
	sort.Strings(p.Addresses)
	return fmt.Sprintf("%s_%v_%v_%s_%s_%s_%s_%s_%s", p.NodeName, p.Addresses, p.Port, p.Protocol, p.ServiceName, p.PodName, p.IngressName, p.Hostname, p.Path)
}

func getNodeNameToMachine(clusterName string, machineLister managementv3.NodeLister, nodeLister v1.NodeLister) (map[string]*managementv3.Node, error) {
	machines, err := machineLister.List(clusterName, labels.NewSelector())
	if err != nil {
		return nil, err
	}
	machineMap := map[string]*managementv3.Node{}
	nodes, err := nodeLister.List("", labels.NewSelector())
	if err != nil {
		return nil, err
	}
	for _, machine := range machines {
		for _, node := range nodes {
			if nodehelper.IsNodeForNode(node, machine) {
				machineMap[node.Name] = machine
			}
		}
	}
	return machineMap, nil
}

func isMachineReady(machine *managementv3.Node) bool {
	for _, cond := range machine.Status.InternalNodeStatus.Conditions {
		if cond.Type == corev1.NodeReady {
			return cond.Status == corev1.ConditionTrue
		}
	}
	return false
}

func getAllNodesPublicEndpointIP(machineLister managementv3.NodeLister, clusterName string) (string, error) {
	var addresses []net.IP
	machines, err := machineLister.List(clusterName, labels.NewSelector())
	if err != nil {
		return "", err
	}
	for _, machine := range machines {
		if machine.Spec.Worker && isMachineReady(machine) {
			nodePublicIP := net.ParseIP(nodehelper.GetEndpointNodeIP(machine))
			if nodePublicIP.String() != "" {
				addresses = append(addresses, nodePublicIP)
			}
		}
	}
	if len(addresses) == 0 {
		return "", nil
	}

	sort.Slice(addresses, func(i, j int) bool {
		return bytes.Compare(addresses[i], addresses[j]) < 0
	})

	return addresses[0].String(), nil
}

func convertIngressToServicePublicEndpointsMap(obj *extensionsv1beta1.Ingress, allNodes bool, allNodesIP string) map[string][]v3.PublicEndpoint {
	var addresses []string
	epsMap := map[string][]v3.PublicEndpoint{}
	ipDomain := settings.IngressIPDomain.Get()
	if !allNodes {
		for _, address := range obj.Status.LoadBalancer.Ingress {
			addresses = append(addresses, address.IP)
		}

	} else {
		if allNodesIP != "" {
			addresses = append(addresses, allNodesIP)
		}
	}

	if len(addresses) == 0 {
		return epsMap
	}

	tlsHosts := sets.NewString()
	for _, t := range obj.Spec.TLS {
		tlsHosts.Insert(t.Hosts...)
	}

	ports := map[int32]string{80: "HTTP", 443: "HTTPS"}
	for _, rule := range obj.Spec.Rules {
		//If the hostname is auto-generated, the public endpoint should be shown only when the
		//hostname is done auto-generation
		if rule.Host == ipDomain {
			continue
		}
		for _, path := range rule.HTTP.Paths {
			for port, proto := range ports {
				if port == 80 {
					if tlsHosts.Has(rule.Host) {
						continue
					}
				} else {
					if !tlsHosts.Has(rule.Host) && rule.Host != "" {
						continue
					}
				}
				p := v3.PublicEndpoint{
					Hostname:    rule.Host,
					Path:        path.Path,
					ServiceName: fmt.Sprintf("%s:%s", obj.Namespace, path.Backend.ServiceName),
					Addresses:   addresses,
					Port:        port,
					Protocol:    proto,
					AllNodes:    allNodes,
					IngressName: fmt.Sprintf("%s:%s", obj.Namespace, obj.Name),
				}
				epsMap[path.Backend.ServiceName] = append(epsMap[path.Backend.ServiceName], p)
			}
		}
	}
	return epsMap
}

func convertIngressToPublicEndpoints(obj *extensionsv1beta1.Ingress, isRKE bool, allNodesIP string) []v3.PublicEndpoint {
	epsMap := convertIngressToServicePublicEndpointsMap(obj, isRKE, allNodesIP)
	var eps []v3.PublicEndpoint
	for _, v := range epsMap {
		eps = append(eps, v...)
	}
	return eps
}
<|endoftext|>


<commit_before>package deplactive

import (
	"encoding/json"
	"io/ioutil"

	"github.com/containerum/chkit/pkg/model/deployment"
)

func fromFile(filename string) (deployment.Deployment, error) {
	depl := deployment.Deployment{}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return depl, err
	}
	kubeDepl := depl.ToKube()
	err = json.Unmarshal(data, &kubeDepl)
	if err != nil {
		return depl, err
	}
	depl = deployment.DeploymentFromKube(kubeDepl)
	return depl, nil
}
<commit_msg>refactor from file func<commit_after>package deplactive

import (
	"encoding/json"
	"io/ioutil"

	"github.com/containerum/chkit/pkg/model/deployment"
)

func FromFile(filename string) (deployment.Deployment, error) {
	depl := deployment.Deployment{}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return depl, err
	}
	kubeDepl := depl.ToKube()
	err = json.Unmarshal(data, &kubeDepl)
	if err != nil {
		return depl, err
	}
	depl = deployment.DeploymentFromKube(kubeDepl)
	return depl, nil
}
<|endoftext|>


<commit_before>package notifiers

import (
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/alerting"
)

func init() {
	alerting.RegisterNotifier(&alerting.NotifierPlugin{
		Type:        "dingding",
		Name:        "DingDing",
		Description: "Sends HTTP POST request to DingDing",
		Factory:     NewDingDingNotifier,
		OptionsTemplate: `

DingDing settings

        Url
        <input type="text" required class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxx"></input>

    `,
	})

}

func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
	url := model.Settings.Get("url").MustString()
	if url == "" {
		return nil, alerting.ValidationError{Reason: "Could not find url property in settings"}
	}

	return &DingDingNotifier{
		NotifierBase: NewNotifierBase(model),
		Url:          url,
		log:          log.New("alerting.notifier.dingding"),
	}, nil
}

type DingDingNotifier struct {
	NotifierBase
	Url string
	log log.Logger
}

func (this *DingDingNotifier) Notify(evalContext *alerting.EvalContext) error {
	this.log.Info("Sending dingding")

	messageUrl, err := evalContext.GetRuleUrl()
	if err != nil {
		this.log.Error("Failed to get messageUrl", "error", err, "dingding", this.Name)
		messageUrl = ""
	}
	this.log.Info("messageUrl:" + messageUrl)

	message := evalContext.Rule.Message
	picUrl := evalContext.ImagePublicUrl
	title := evalContext.GetNotificationTitle()

	bodyJSON, err := simplejson.NewJson([]byte(`{
		"msgtype": "link",
		"link": {
			"text": "` + message + `",
			"title": "` + title + `",
			"picUrl": "` + picUrl + `",
			"messageUrl": "` + messageUrl + `"
		}
	}`))

	if err != nil {
		this.log.Error("Failed to create Json data", "error", err, "dingding", this.Name)
	}

	body, err := bodyJSON.MarshalJSON()
	if err != nil {
		return err
	}

	cmd := &m.SendWebhookSync{
		Url:  this.Url,
		Body: string(body),
	}

	if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
		this.log.Error("Failed to send DingDing", "error", err, "dingding", this.Name)
		return err
	}

	return nil
}
<commit_msg>fix cannot receive dingding alert bug<commit_after>package notifiers

import (
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/alerting"
)

func init() {
	alerting.RegisterNotifier(&alerting.NotifierPlugin{
		Type:        "dingding",
		Name:        "DingDing",
		Description: "Sends HTTP POST request to DingDing",
		Factory:     NewDingDingNotifier,
		OptionsTemplate: `

DingDing settings

        Url
        <input type="text" required class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxx"></input>

    `,
	})

}

func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
	url := model.Settings.Get("url").MustString()
	if url == "" {
		return nil, alerting.ValidationError{Reason: "Could not find url property in settings"}
	}

	return &DingDingNotifier{
		NotifierBase: NewNotifierBase(model),
		Url:          url,
		log:          log.New("alerting.notifier.dingding"),
	}, nil
}

type DingDingNotifier struct {
	NotifierBase
	Url string
	log log.Logger
}

func (this *DingDingNotifier) Notify(evalContext *alerting.EvalContext) error {
	this.log.Info("Sending dingding")

	messageUrl, err := evalContext.GetRuleUrl()
	if err != nil {
		this.log.Error("Failed to get messageUrl", "error", err, "dingding", this.Name)
		messageUrl = ""
	}
	this.log.Info("messageUrl:" + messageUrl)

	message := evalContext.Rule.Message
	picUrl := evalContext.ImagePublicUrl
	title := evalContext.GetNotificationTitle()
	if message == "" {
		message = title
	}

	bodyJSON, err := simplejson.NewJson([]byte(`{
		"msgtype": "link",
		"link": {
			"text": "` + message + `",
			"title": "` + title + `",
			"picUrl": "` + picUrl + `",
			"messageUrl": "` + messageUrl + `"
		}
	}`))

	if err != nil {
		this.log.Error("Failed to create Json data", "error", err, "dingding", this.Name)
	}

	body, err := bodyJSON.MarshalJSON()
	if err != nil {
		return err
	}

	cmd := &m.SendWebhookSync{
		Url:  this.Url,
		Body: string(body),
	}

	if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
		this.log.Error("Failed to send DingDing", "error", err, "dingding", this.Name)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// APTFixityChecker performs ongoing fixity checks on files stored in S3.
type APTFixityChecker struct {
	// Context contains contextual info, such as a reference to the logger,
	// a Pharos client, basic configuration data and other things the fixity
	// checker needs to do its work.
	Context *context.Context
	// FixityChannel is where we calculate Sha256 digests of files stored in S3.
	FixityChannel chan *models.FixityResult
	// RecordChannel is where record PremisEvents in Pharos.
	RecordChannel chan *models.FixityResult
	// PostProcessChannel is where we dispose of NSQ messages and log outcomes.
	PostProcessChannel chan *models.FixityResult
	// ItemsInProcess contains a map of items we're currently processing.
	// When there's a backlog, it's common for items to be queued more than
	// once for fixity checking. We don't want to perform the fixity check
	// if it's already underway.
	ItemsInProcess *models.SynchronizedMap
}

func NewAPTFixityChecker(_context *context.Context) *APTFixityChecker {
	checker := &APTFixityChecker{
		Context:        _context,
		ItemsInProcess: models.NewSynchronizedMap(),
	}
	workerBufferSize := _context.Config.FixityWorker.Workers * 10
	checker.FixityChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.RecordChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.PostProcessChannel = make(chan *models.FixityResult, workerBufferSize)
	for i := 0; i < _context.Config.StoreWorker.Workers; i++ {
		go checker.checkFixity()
		go checker.record()
		go checker.postProcess()
	}
	return checker
}

// HandleMessage handles a new message from NSQ. Unlike most other NSQ messages,
// where the message.Body is a WorkItem.Id (int as string), messages in the
// apt_fixity queue contain a GenericFile.Identifier. So the entire message body
// will be something like "georgetown.edu/georgetown.edu.10822_707412".
func (checker *APTFixityChecker) HandleMessage(message *nsq.Message) error {
	fixityResult := checker.buildFixityResult(message)
	if fixityResult.Error != nil {
		checker.Context.MessageLog.Error("Cannot process %s: %v",
			string(message.Body), fixityResult.Error.Error())
		message.Finish()
		return nil // Should we return an error to NSQ?
	}

	// Item may have been queued multiple times and then checked a few hours ago.
	if !checker.stillNeedsFixityCheck(fixityResult.GenericFile) {
		checker.Context.MessageLog.Info("Skipping %s because it had a fixity check at %s.",
			fixityResult.GenericFile.Identifier,
			fixityResult.GenericFile.LastFixityCheck.Format(time.RFC3339))
		message.Finish()
		return nil
	}

	// Check syncmap to see if this item is already in process.
	startedAt := checker.ItemsInProcess.Get(fixityResult.GenericFile.Identifier)
	if startedAt != "" {
		checker.Context.MessageLog.Info("Skipping %s: already in process as of %s.",
			fixityResult.GenericFile.Identifier, startedAt)
		message.Finish()
		return nil
	}

	// Note that we're working on this.
	checker.ItemsInProcess.Add(fixityResult.GenericFile.Identifier, time.Now().UTC().Format(time.RFC3339))
	checker.Context.MessageLog.Info("Added %s to items in process", fixityResult.GenericFile.Identifier)

	checker.Context.MessageLog.Info("Putting %s into fixity channel",
		fixityResult.GenericFile.Identifier)

	checker.FixityChannel <- fixityResult
	return nil
}

// checkFixity calls the downloader to calculate the sha256 digest of the
// file in S3.
func (checker *APTFixityChecker) checkFixity() {
	for fixityResult := range checker.FixityChannel {
		// Here's where we do the actual digest calculation.
		checker.getFixityValueOfS3File(fixityResult)
		if fixityResult.Error != nil {
			checker.PostProcessChannel <- fixityResult
		} else {
			checker.RecordChannel <- fixityResult
		}
	}
}

// record records a PremisEvent in Pharos saying when this fixity check
// was performed and whether it succeeded.
func (checker *APTFixityChecker) record() {
	for fixityResult := range checker.RecordChannel {
		// Create PREMIS event saying whether fixity event
		// succeeded or failed.
		event, err := models.NewEventGenericFileFixityCheck(
			time.Now().UTC(),
			constants.AlgSha256,
			fixityResult.Sha256,
			fixityResult.Sha256 == fixityResult.PharosSha256())
		if err != nil {
			fixityResult.Error = fmt.Errorf("Could not create Premis Event for %s: %v",
				fixityResult.GenericFile.Identifier, err)
		} else {
			resp := checker.Context.PharosClient.PremisEventSave(event)
			if resp.Error != nil {
				fixityResult.Error = fmt.Errorf("After completing fixity check for %s, "+
					"could not save PremisEvent to Pharos: %v. Event data: %v",
					fixityResult.GenericFile.Identifier, resp.Error, event)
			} else {
				checker.Context.MessageLog.Info("Completing fixity check for %s, "+
					"and saved PremisEvent %s to Pharos",
					fixityResult.GenericFile.Identifier, event.Identifier)
			}
		}
		checker.PostProcessChannel <- fixityResult
	}
}

// postProcess does some logging and tells NSQ to either finish
// the message or requeue it.
func (checker *APTFixityChecker) postProcess() {
	for fixityResult := range checker.PostProcessChannel {
		// Finish or requeue NSQ
		if fixityResult.Error != nil {
			if fixityResult.ErrorIsFatal {
				checker.Context.MessageLog.Error("%s (FATAL)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Finish()
			} else {
				checker.Context.MessageLog.Error("%s (transient)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Requeue(1 * time.Minute)
			}
		} else {
			if fixityResult.PharosSha256() == fixityResult.Sha256 {
				checker.Context.MessageLog.Info("Fixity check complete for %s. Fixity %s matches.",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256)
			} else {
				checker.Context.MessageLog.Warning("Fixity check complete for %s. S3 fixity %s "+
					"DOES NOT MATCH PHAROS FIXITY %s",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256,
					fixityResult.PharosSha256())
			}
			fixityResult.NSQMessage.Finish()
		}
		checker.ItemsInProcess.Delete(fixityResult.GenericFile.Identifier)
		checker.Context.MessageLog.Info("Removed %s from items in process", fixityResult.GenericFile.Identifier)
	}
}

// getFixityValueOfS3File calculates the sha256 digest of an S3 file.
// The downloader streams the file from S3 to /dev/null, because
// we don't need to have the file on disk. We can calculate the
// digest from the stream. We get the file from S3/Virginia, not
// Glacier/Oregon! When this is done, the fixity value will be in
// fixityResult.Sha256.
func (checker *APTFixityChecker) getFixityValueOfS3File(fixityResult *models.FixityResult) {
	bucket, key, err := fixityResult.BucketAndKey()
	if err != nil {
		fixityResult.Error = fmt.Errorf("Can't get S3 bucket and key names for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		fixityResult.ErrorIsFatal = true
		return
	}
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		bucket,      // should be S3 preservation bucket
		key,         // s3 key to fetch
		"/dev/null", // local path at which to save the s3 file
		false,       // don't calculate md5 digest
		true)        // do calculate sha256 digest
	downloader.Fetch()
	if downloader.ErrorMessage != "" {
		fixityResult.Error = fmt.Errorf("Error fetching file %s (%s/%s) from S3: %s",
			fixityResult.GenericFile.Identifier, bucket, key,
			downloader.ErrorMessage)
		if strings.Contains(downloader.ErrorMessage, "NoSuchKey") {
			fixityResult.ErrorIsFatal = true
		}
		return
	}
	fixityResult.S3FileExists = true
	fixityResult.Sha256 = downloader.Sha256Digest
	return
}

// buildFixityResult builds the manifest that we'll need to record
// the fixity check process and its outcome.
func (checker *APTFixityChecker) buildFixityResult(message *nsq.Message) *models.FixityResult {
	fixityResult := models.NewFixityResult(message)
	gfIdentifier := strings.TrimSpace(string(message.Body))
	// Get GenericFile with checksums (param includeRelations = true)
	resp := checker.Context.PharosClient.GenericFileGet(gfIdentifier, true)
	if resp.Error != nil {
		fixityResult.Error = fmt.Errorf("Can't get generic file %s from Pharos: %v", resp.Error.Error())
		if resp.Response.StatusCode == 404 {
			fixityResult.ErrorIsFatal = true
		}
		return fixityResult
	}
	fixityResult.GenericFile = resp.GenericFile()
	if fixityResult.GenericFile.URI == "" {
		fixityResult.Error = fmt.Errorf("GenericFile %s has no S3 URI.", fixityResult.GenericFile.Identifier)
		fixityResult.ErrorIsFatal = true
	}
	return fixityResult
}

// stillNeedsFixityCheck returns true if the GenericFile still needs a fixity check.
// When we have a backlog of items to check, we may run into the following scenario:
//
// - apt_queue finds a GenericFile that hasn't been checked in 90 days,
//   so it adds it to the NSQ fixity check topic
// - there are already 10,000 items in that topic
// - apt_queue runs again an hour later and adds the same file to the queue
// - the fixity checker finally gets around to checking that file
// - hours later, the fixity checker encounters that file once again in the queue
//
// In this common case, the fixity checker should just mark the item as finished
// and move on if it sees that the item has been checked recently.
func (checker *APTFixityChecker) stillNeedsFixityCheck(gf *models.GenericFile) bool {
	hoursSinceLastFixityCheck := time.Since(gf.LastFixityCheck).Hours()
	maxHoursAllowed := float64(checker.Context.Config.MaxDaysSinceFixityCheck * 24)
	return hoursSinceLastFixityCheck >= maxHoursAllowed
}
<commit_msg>Set file/object identifiers on fixity check event before saving<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// APTFixityChecker performs ongoing fixity checks on files stored in S3.
type APTFixityChecker struct {
	// Context contains contextual info, such as a reference to the logger,
	// a Pharos client, basic configuration data and other things the fixity
	// checker needs to do its work.
	Context *context.Context
	// FixityChannel is where we calculate Sha256 digests of files stored in S3.
	FixityChannel chan *models.FixityResult
	// RecordChannel is where record PremisEvents in Pharos.
	RecordChannel chan *models.FixityResult
	// PostProcessChannel is where we dispose of NSQ messages and log outcomes.
	PostProcessChannel chan *models.FixityResult
	// ItemsInProcess contains a map of items we're currently processing.
	// When there's a backlog, it's common for items to be queued more than
	// once for fixity checking. We don't want to perform the fixity check
	// if it's already underway.
	ItemsInProcess *models.SynchronizedMap
}

func NewAPTFixityChecker(_context *context.Context) *APTFixityChecker {
	checker := &APTFixityChecker{
		Context:        _context,
		ItemsInProcess: models.NewSynchronizedMap(),
	}
	workerBufferSize := _context.Config.FixityWorker.Workers * 10
	checker.FixityChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.RecordChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.PostProcessChannel = make(chan *models.FixityResult, workerBufferSize)
	for i := 0; i < _context.Config.StoreWorker.Workers; i++ {
		go checker.checkFixity()
		go checker.record()
		go checker.postProcess()
	}
	return checker
}

// HandleMessage handles a new message from NSQ. Unlike most other NSQ messages,
// where the message.Body is a WorkItem.Id (int as string), messages in the
// apt_fixity queue contain a GenericFile.Identifier. So the entire message body
// will be something like "georgetown.edu/georgetown.edu.10822_707412".
func (checker *APTFixityChecker) HandleMessage(message *nsq.Message) error {
	fixityResult := checker.buildFixityResult(message)
	if fixityResult.Error != nil {
		checker.Context.MessageLog.Error("Cannot process %s: %v",
			string(message.Body), fixityResult.Error.Error())
		message.Finish()
		return nil // Should we return an error to NSQ?
	}

	// Item may have been queued multiple times and then checked a few hours ago.
	if !checker.stillNeedsFixityCheck(fixityResult.GenericFile) {
		checker.Context.MessageLog.Info("Skipping %s because it had a fixity check at %s.",
			fixityResult.GenericFile.Identifier,
			fixityResult.GenericFile.LastFixityCheck.Format(time.RFC3339))
		message.Finish()
		return nil
	}

	// Check syncmap to see if this item is already in process.
	startedAt := checker.ItemsInProcess.Get(fixityResult.GenericFile.Identifier)
	if startedAt != "" {
		checker.Context.MessageLog.Info("Skipping %s: already in process as of %s.",
			fixityResult.GenericFile.Identifier, startedAt)
		message.Finish()
		return nil
	}

	// Note that we're working on this.
	checker.ItemsInProcess.Add(fixityResult.GenericFile.Identifier, time.Now().UTC().Format(time.RFC3339))
	checker.Context.MessageLog.Info("Added %s to items in process", fixityResult.GenericFile.Identifier)

	checker.Context.MessageLog.Info("Putting %s into fixity channel",
		fixityResult.GenericFile.Identifier)

	checker.FixityChannel <- fixityResult
	return nil
}

// checkFixity calls the downloader to calculate the sha256 digest of the
// file in S3.
func (checker *APTFixityChecker) checkFixity() {
	for fixityResult := range checker.FixityChannel {
		// Here's where we do the actual digest calculation.
		checker.getFixityValueOfS3File(fixityResult)
		if fixityResult.Error != nil {
			checker.PostProcessChannel <- fixityResult
		} else {
			checker.RecordChannel <- fixityResult
		}
	}
}

// record records a PremisEvent in Pharos saying when this fixity check
// was performed and whether it succeeded.
func (checker *APTFixityChecker) record() {
	for fixityResult := range checker.RecordChannel {
		// Create PREMIS event saying whether fixity event
		// succeeded or failed.
		event, err := models.NewEventGenericFileFixityCheck(
			time.Now().UTC(),
			constants.AlgSha256,
			fixityResult.Sha256,
			fixityResult.Sha256 == fixityResult.PharosSha256())
		if err != nil {
			fixityResult.Error = fmt.Errorf("Could not create Premis Event for %s: %v",
				fixityResult.GenericFile.Identifier, err)
		} else {
			event.IntellectualObjectId = fixityResult.GenericFile.IntellectualObjectId
			event.IntellectualObjectIdentifier = fixityResult.GenericFile.IntellectualObjectIdentifier
			event.GenericFileId = fixityResult.GenericFile.Id
			event.GenericFileIdentifier = fixityResult.GenericFile.Identifier
			resp := checker.Context.PharosClient.PremisEventSave(event)
			if resp.Error != nil {
				fixityResult.Error = fmt.Errorf("After completing fixity check for %s, "+
					"could not save PremisEvent to Pharos: %v. Event data: %v",
					fixityResult.GenericFile.Identifier, resp.Error, event)
			} else {
				checker.Context.MessageLog.Info("Completing fixity check for %s, "+
					"and saved PremisEvent %s to Pharos",
					fixityResult.GenericFile.Identifier, event.Identifier)
			}
		}
		checker.PostProcessChannel <- fixityResult
	}
}

// postProcess does some logging and tells NSQ to either finish
// the message or requeue it.
func (checker *APTFixityChecker) postProcess() {
	for fixityResult := range checker.PostProcessChannel {
		// Finish or requeue NSQ
		if fixityResult.Error != nil {
			if fixityResult.ErrorIsFatal {
				checker.Context.MessageLog.Error("%s (FATAL)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Finish()
			} else {
				checker.Context.MessageLog.Error("%s (transient)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Requeue(1 * time.Minute)
			}
		} else {
			if fixityResult.PharosSha256() == fixityResult.Sha256 {
				checker.Context.MessageLog.Info("Fixity check complete for %s. Fixity %s matches.",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256)
			} else {
				checker.Context.MessageLog.Warning("Fixity check complete for %s. S3 fixity %s "+
					"DOES NOT MATCH PHAROS FIXITY %s",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256,
					fixityResult.PharosSha256())
			}
			fixityResult.NSQMessage.Finish()
		}
		checker.ItemsInProcess.Delete(fixityResult.GenericFile.Identifier)
		checker.Context.MessageLog.Info("Removed %s from items in process", fixityResult.GenericFile.Identifier)
	}
}

// getFixityValueOfS3File calculates the sha256 digest of an S3 file.
// The downloader streams the file from S3 to /dev/null, because
// we don't need to have the file on disk. We can calculate the
// digest from the stream. We get the file from S3/Virginia, not
// Glacier/Oregon! When this is done, the fixity value will be in
// fixityResult.Sha256.
func (checker *APTFixityChecker) getFixityValueOfS3File(fixityResult *models.FixityResult) {
	bucket, key, err := fixityResult.BucketAndKey()
	if err != nil {
		fixityResult.Error = fmt.Errorf("Can't get S3 bucket and key names for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		fixityResult.ErrorIsFatal = true
		return
	}
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		bucket,      // should be S3 preservation bucket
		key,         // s3 key to fetch
		"/dev/null", // local path at which to save the s3 file
		false,       // don't calculate md5 digest
		true)        // do calculate sha256 digest
	downloader.Fetch()
	if downloader.ErrorMessage != "" {
		fixityResult.Error = fmt.Errorf("Error fetching file %s (%s/%s) from S3: %s",
			fixityResult.GenericFile.Identifier, bucket, key,
			downloader.ErrorMessage)
		if strings.Contains(downloader.ErrorMessage, "NoSuchKey") {
			fixityResult.ErrorIsFatal = true
		}
		return
	}
	fixityResult.S3FileExists = true
	fixityResult.Sha256 = downloader.Sha256Digest
	return
}

// buildFixityResult builds the manifest that we'll need to record
// the fixity check process and its outcome.
func (checker *APTFixityChecker) buildFixityResult(message *nsq.Message) *models.FixityResult {
	fixityResult := models.NewFixityResult(message)
	gfIdentifier := strings.TrimSpace(string(message.Body))
	// Get GenericFile with checksums (param includeRelations = true)
	resp := checker.Context.PharosClient.GenericFileGet(gfIdentifier, true)
	if resp.Error != nil {
		fixityResult.Error = fmt.Errorf("Can't get generic file %s from Pharos: %v", resp.Error.Error())
		if resp.Response.StatusCode == 404 {
			fixityResult.ErrorIsFatal = true
		}
		return fixityResult
	}
	fixityResult.GenericFile = resp.GenericFile()
	if fixityResult.GenericFile.URI == "" {
		fixityResult.Error = fmt.Errorf("GenericFile %s has no S3 URI.", fixityResult.GenericFile.Identifier)
		fixityResult.ErrorIsFatal = true
	}
	return fixityResult
}

// stillNeedsFixityCheck returns true if the GenericFile still needs a fixity check.
// When we have a backlog of items to check, we may run into the following scenario:
//
// - apt_queue finds a GenericFile that hasn't been checked in 90 days,
//   so it adds it to the NSQ fixity check topic
// - there are already 10,000 items in that topic
// - apt_queue runs again an hour later and adds the same file to the queue
// - the fixity checker finally gets around to checking that file
// - hours later, the fixity checker encounters that file once again in the queue
//
// In this common case, the fixity checker should just mark the item as finished
// and move on if it sees that the item has been checked recently.
func (checker *APTFixityChecker) stillNeedsFixityCheck(gf *models.GenericFile) bool {
	hoursSinceLastFixityCheck := time.Since(gf.LastFixityCheck).Hours()
	maxHoursAllowed := float64(checker.Context.Config.MaxDaysSinceFixityCheck * 24)
	return hoursSinceLastFixityCheck >= maxHoursAllowed
}
<|endoftext|>