<commit_before>// Copyright 2017 ETH Zurich
// Copyright 2018 ETH Zurich, Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package session

import (
	"time"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/ctrl"
	"github.com/scionproto/scion/go/lib/infra"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/spath"
	"github.com/scionproto/scion/go/sig/disp"
	"github.com/scionproto/scion/go/sig/egress"
	"github.com/scionproto/scion/go/sig/mgmt"
	"github.com/scionproto/scion/go/sig/sigcmn"
	"github.com/scionproto/scion/go/sig/siginfo"
)

const (
	// How long before path TTL expires we should already try to switch to a different path.
	tickLen       = 500 * time.Millisecond
	tout          = 1 * time.Second
	writeTout     = 100 * time.Millisecond
	pathExpiryLen = 10 * time.Second
)

// sessMonitor is responsible for monitoring a session, polling remote SIGs, and switching
// remote SIGs and paths as needed.
type sessMonitor struct {
	log.Logger
	// the Session this instance is monitoring.
	sess *Session
	// the (filtered) pool of paths to the remote AS, maintained by pathmgr.
	pool egress.PathPool
	// the pool of paths this session is currently using, frequently refreshed from pool.
	sessPathPool egress.SessPathPool
	// the remote Info (remote SIG, and path) used for sending polls. This
	// differs from the parent session's remote info when the session monitor
	// is polling a new SIG or over a new path, and is waiting for a response.
	smRemote *egress.RemoteInfo
	// when sessMonitor is trying to switch SIGs/paths, this is the id of the
	// last PollReq sent, so that sessMonitor can correlate replies to the
	// remoteInfo used for the request.
	updateMsgId mgmt.MsgIdType
	// the last time a PollRep was received.
	lastReply time.Time
}

func newSessMonitor(sess *Session) *sessMonitor {
	return &sessMonitor{
		Logger: sess.Logger, sess: sess, pool: sess.pool, sessPathPool: make(egress.SessPathPool),
	}
}

func (sm *sessMonitor) run() {
	defer close(sm.sess.sessMonStopped)
	// Setup timers
	reqTick := time.NewTicker(tickLen)
	defer reqTick.Stop()
	pathExpiryTick := time.NewTicker(pathExpiryLen)
	defer pathExpiryTick.Stop()
	// Register with SIG ctrl dispatcher
	regc := make(disp.RegPldChan, 1)
	disp.Dispatcher.Register(disp.RegPollRep, disp.MkRegPollKey(sm.sess.IA(), sm.sess.SessId), regc)
	sm.lastReply = time.Now()
	// Start by querying for the remote SIG instance.
	sm.smRemote = &egress.RemoteInfo{
		Sig: &siginfo.Sig{
			IA:   sm.sess.IA(),
			Host: addr.SvcSIG,
		},
		SessPath: sm.sessPathPool.Get(""),
	}
Top:
	for {
		select {
		case <-sm.sess.sessMonStop:
			break Top
		case <-reqTick.C:
			// Update paths and sigs
			sm.sessPathPool.Update(sm.pool.Paths())
			sm.updateRemote()
			sm.sendReq()
		case rpld := <-regc:
			sm.handleRep(rpld)
		case <-pathExpiryTick.C:
			for _, path := range sm.sessPathPool {
				path.ExpireFails()
			}
		}
	}
	err := disp.Dispatcher.Unregister(disp.RegPollRep, disp.MkRegPollKey(sm.sess.IA(),
		sm.sess.SessId))
	if err != nil {
		log.Error("sessMonitor: unable to unregister from ctrl dispatcher", "err", err)
	}
	sm.Info("sessMonitor: stopped")
}

func (sm *sessMonitor) updateRemote() {
	// There were no replies from the remote SIG for some time. We don't know whether
	// the failure was caused by bad path or bad SIG. Therefore, we choose a different
	// path but also ask for a new SIG address via anycast SvcSIG request.
	since := time.Since(sm.lastReply)
	if since > tout {
		// FIXME(kormat): these debug statements should be converted to prom metrics.
		sm.Info("sessMonitor: Remote SIG timeout", "remote", sm.smRemote, "duration", since)
		sm.sess.healthy.Store(false)
		if sm.smRemote.SessPath != nil {
			// Update path statistics. This is a bit of a stretch. The path
			// may be OK, but the remote SIG may be down. However, we accept
			// the inaccuracy so that we don't have to do separate health
			// checking for the path.
			sm.smRemote.SessPath.Fail()
		}
		// Start monitoring new path and discover a new SIG.
		sm.smRemote.Sig.Host = addr.SvcSIG
		sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		// XXX(roosd): The session's remote SIG will remain the same until the
		// monitor discovers a remote SIG.
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}

	// There's no path selected yet. This happens at the beginning of the session,
	// but also when the pool is empty. Try to get a new path.
	if sm.smRemote.SessPath == nil {
		sm.Info("sessMonitor: Path not available", "remote", sm.smRemote)
		sm.sess.healthy.Store(false)
		// Start monitoring the new path.
		sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}

	// The current path was retired from the path pool. Traffic must no longer be sent on
	// the old path. This implies that the encap traffic is sent on a path that has not been
	// tested by the session monitor yet. If the new path is unhealthy, it is changed quickly
	// by the session monitor through the regular timeout mechanism above.
	updatedPath, ok := sm.sessPathPool[sm.smRemote.SessPath.Key()]
	if !ok {
		sm.Info("sessMonitor: Current path was invalidated", "remote", sm.smRemote)
		// Start monitoring the new path.
		sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		// Make session use the new path immediately even though we haven't yet checked
		// whether it works.
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}

	// If the current path is about to expire, make session use the updated version of the path.
	// If the updated version is about to expire as well, let's switch to a different path.
	if sm.smRemote.SessPath.IsCloseToExpiry() {
		sm.Info("sessMonitor: Current path is about to expire", "remote", sm.smRemote)
		sm.smRemote.SessPath = updatedPath
		if sm.smRemote.SessPath.IsCloseToExpiry() {
			sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		}
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}
}

// updateSessSnap updates the remote snapshot in the session. If the new remote
// SIG host is an SVC address, the previous host of the session is kept.
func (sm *sessMonitor) updateSessSnap() {
	// Copy the remote to avoid capturing the object in the session.
	remote := *sm.smRemote
	// XXX(roosd): Data traffic should never be sent to a SVC address.
	if remote.Sig.Host.Equal(addr.SvcSIG) {
		remote.Sig = sm.sess.Remote().Sig
	}
	sm.sess.currRemote.Store(&remote)
}

func (sm *sessMonitor) getNewPath(old *egress.SessPath) *egress.SessPath {
	if old == nil {
		return sm.sessPathPool.Get("")
	}
	return sm.sessPathPool.Get(old.Key())
}

func (sm *sessMonitor) sendReq() {
	if sm.smRemote == nil || sm.smRemote.SessPath == nil {
		return
	}
	sm.updateMsgId = mgmt.MsgIdType(time.Now().UnixNano())
	spld, err := mgmt.NewPld(sm.updateMsgId, mgmt.NewPollReq(sigcmn.MgmtAddr, sm.sess.SessId))
	if err != nil {
		sm.Error("sessMonitor: Error creating SIGCtrl payload", "err", err)
		return
	}
	cpld, err := ctrl.NewPld(spld, nil)
	if err != nil {
		sm.Error("sessMonitor: Error creating Ctrl payload", "err", err)
		return
	}
	scpld, err := cpld.SignedPld(infra.NullSigner)
	if err != nil {
		sm.Error("sessMonitor: Error creating signed Ctrl payload", "err", err)
		return
	}
	raw, err := scpld.PackPld()
	if err != nil {
		sm.Error("sessMonitor: Error packing signed Ctrl payload", "err", err)
		return
	}
	raddr := sm.smRemote.Sig.CtrlSnetAddr()
	raddr.Path = spath.New(sm.smRemote.SessPath.PathEntry().Path.FwdPath)
	if err := raddr.Path.InitOffsets(); err != nil {
		sm.Error("sessMonitor: Error initializing path offsets", "err", err)
	}
	nh, err := sm.smRemote.SessPath.PathEntry().HostInfo.Overlay()
	if err != nil {
		sm.Error("sessMonitor: Unsupported NextHop", "err", err)
	}
	raddr.NextHop = nh
	// XXX(kormat): if this blocks, both the sessMon and egress worker
	// goroutines will block. Can't just use SetWriteDeadline, as both
	// goroutines write to it.
	_, err = sm.sess.conn.WriteToSCION(raw, raddr)
	if err != nil {
		sm.Error("sessMonitor: Error sending signed Ctrl payload", "err", err)
	}
}

func (sm *sessMonitor) handleRep(rpld *disp.RegPld) {
	pollRep, ok := rpld.P.(*mgmt.PollRep)
	if !ok {
		sm.Error("sessMonitor: non-SIGPollRep payload received",
			"src", rpld.Addr, "type", common.TypeOf(rpld.P), "pld", rpld.P)
		return
	}
	if !sm.sess.IA().Equal(rpld.Addr.IA) {
		sm.Error("sessMonitor: SIGPollRep from wrong IA",
			"expected", sm.sess.IA(), "actual", rpld.Addr.IA)
		return
	}
	// Only update the session's RemoteInfo if we get a response matching
	// the last poll we sent.
	if sm.updateMsgId == rpld.Id {
		sm.lastReply = time.Now()
		// Update sessmon's remote.
		sm.smRemote.Sig = &siginfo.Sig{
			IA:          sm.smRemote.Sig.IA,
			Host:        pollRep.Addr.Ctrl.Host(),
			CtrlL4Port:  int(pollRep.Addr.Ctrl.Port),
			EncapL4Port: int(pollRep.Addr.EncapPort),
		}
		// Update session's remote, if needed.
		sessRemote := sm.sess.Remote()
		if sessRemote == nil || !sm.smRemote.Sig.Equal(sessRemote.Sig) {
			sm.updateSessSnap()
			sm.Info("sessMonitor: updating remote Info", "msgId", rpld.Id, "remote", sm.smRemote)
		}
		sm.sess.healthy.Store(true)
	} else {
		// This is going to happen if latency of the path is greater than the poll ticker period.
		// TODO(sustrik): We should monitor this to spot paths where the latency is high enough to
		// to disrupt orderly SIG operation.
		sm.Info("Reply to an old request received", "request", sm.updateMsgId, "reply", rpld.Id)
	}

}
<commit_msg>SIG: Avoid panic on startup (#2618)<commit_after>// Copyright 2017 ETH Zurich
// Copyright 2018 ETH Zurich, Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package session

import (
	"time"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/ctrl"
	"github.com/scionproto/scion/go/lib/infra"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/spath"
	"github.com/scionproto/scion/go/sig/disp"
	"github.com/scionproto/scion/go/sig/egress"
	"github.com/scionproto/scion/go/sig/mgmt"
	"github.com/scionproto/scion/go/sig/sigcmn"
	"github.com/scionproto/scion/go/sig/siginfo"
)

const (
	// How long before path TTL expires we should already try to switch to a different path.
	tickLen       = 500 * time.Millisecond
	tout          = 1 * time.Second
	writeTout     = 100 * time.Millisecond
	pathExpiryLen = 10 * time.Second
)

// sessMonitor is responsible for monitoring a session, polling remote SIGs, and switching
// remote SIGs and paths as needed.
type sessMonitor struct {
	log.Logger
	// the Session this instance is monitoring.
	sess *Session
	// the (filtered) pool of paths to the remote AS, maintained by pathmgr.
	pool egress.PathPool
	// the pool of paths this session is currently using, frequently refreshed from pool.
	sessPathPool egress.SessPathPool
	// the remote Info (remote SIG, and path) used for sending polls. This
	// differs from the parent session's remote info when the session monitor
	// is polling a new SIG or over a new path, and is waiting for a response.
	smRemote *egress.RemoteInfo
	// when sessMonitor is trying to switch SIGs/paths, this is the id of the
	// last PollReq sent, so that sessMonitor can correlate replies to the
	// remoteInfo used for the request.
	updateMsgId mgmt.MsgIdType
	// the last time a PollRep was received.
	lastReply time.Time
}

func newSessMonitor(sess *Session) *sessMonitor {
	return &sessMonitor{
		Logger: sess.Logger, sess: sess, pool: sess.pool, sessPathPool: make(egress.SessPathPool),
	}
}

func (sm *sessMonitor) run() {
	defer close(sm.sess.sessMonStopped)
	// Setup timers
	reqTick := time.NewTicker(tickLen)
	defer reqTick.Stop()
	pathExpiryTick := time.NewTicker(pathExpiryLen)
	defer pathExpiryTick.Stop()
	// Register with SIG ctrl dispatcher
	regc := make(disp.RegPldChan, 1)
	disp.Dispatcher.Register(disp.RegPollRep, disp.MkRegPollKey(sm.sess.IA(), sm.sess.SessId), regc)
	sm.lastReply = time.Now()
	// Start by querying for the remote SIG instance.
	sm.smRemote = &egress.RemoteInfo{
		Sig: &siginfo.Sig{
			IA:   sm.sess.IA(),
			Host: addr.SvcSIG,
		},
		SessPath: sm.sessPathPool.Get(""),
	}
Top:
	for {
		select {
		case <-sm.sess.sessMonStop:
			break Top
		case <-reqTick.C:
			// Update paths and sigs
			sm.sessPathPool.Update(sm.pool.Paths())
			sm.updateRemote()
			sm.sendReq()
		case rpld := <-regc:
			sm.handleRep(rpld)
		case <-pathExpiryTick.C:
			for _, path := range sm.sessPathPool {
				path.ExpireFails()
			}
		}
	}
	err := disp.Dispatcher.Unregister(disp.RegPollRep, disp.MkRegPollKey(sm.sess.IA(),
		sm.sess.SessId))
	if err != nil {
		log.Error("sessMonitor: unable to unregister from ctrl dispatcher", "err", err)
	}
	sm.Info("sessMonitor: stopped")
}

func (sm *sessMonitor) updateRemote() {
	// There were no replies from the remote SIG for some time. We don't know whether
	// the failure was caused by bad path or bad SIG. Therefore, we choose a different
	// path but also ask for a new SIG address via anycast SvcSIG request.
	since := time.Since(sm.lastReply)
	if since > tout {
		// FIXME(kormat): these debug statements should be converted to prom metrics.
		sm.Info("sessMonitor: Remote SIG timeout", "remote", sm.smRemote, "duration", since)
		sm.sess.healthy.Store(false)
		if sm.smRemote.SessPath != nil {
			// Update path statistics. This is a bit of a stretch. The path
			// may be OK, but the remote SIG may be down. However, we accept
			// the inaccuracy so that we don't have to do separate health
			// checking for the path.
			sm.smRemote.SessPath.Fail()
		}
		// Start monitoring new path and discover a new SIG.
		sm.smRemote.Sig.Host = addr.SvcSIG
		sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		// XXX(roosd): The session's remote SIG will remain the same until the
		// monitor discovers a remote SIG.
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}

	// There's no path selected yet. This happens at the beginning of the session,
	// but also when the pool is empty. Try to get a new path.
	if sm.smRemote.SessPath == nil {
		sm.Info("sessMonitor: Path not available", "remote", sm.smRemote)
		sm.sess.healthy.Store(false)
		// Start monitoring the new path.
		sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}

	// The current path was retired from the path pool. Traffic must no longer be sent on
	// the old path. This implies that the encap traffic is sent on a path that has not been
	// tested by the session monitor yet. If the new path is unhealthy, it is changed quickly
	// by the session monitor through the regular timeout mechanism above.
	updatedPath, ok := sm.sessPathPool[sm.smRemote.SessPath.Key()]
	if !ok {
		sm.Info("sessMonitor: Current path was invalidated", "remote", sm.smRemote)
		// Start monitoring the new path.
		sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		// Make session use the new path immediately even though we haven't yet checked
		// whether it works.
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}

	// If the current path is about to expire, make session use the updated version of the path.
	// If the updated version is about to expire as well, let's switch to a different path.
	if sm.smRemote.SessPath.IsCloseToExpiry() {
		sm.Info("sessMonitor: Current path is about to expire", "remote", sm.smRemote)
		sm.smRemote.SessPath = updatedPath
		if sm.smRemote.SessPath.IsCloseToExpiry() {
			sm.smRemote.SessPath = sm.getNewPath(sm.smRemote.SessPath)
		}
		sm.updateSessSnap()
		sm.Info("sessMonitor: New remote", "remote", sm.smRemote)
		return
	}
}

// updateSessSnap updates the remote snapshot in the session. If the new remote
// SIG host is an SVC address, the previous host of the session is kept.
func (sm *sessMonitor) updateSessSnap() {
	// Copy the remote to avoid capturing the object in the session.
	remote := *sm.smRemote
	// XXX(roosd): Data traffic should never be sent to a SVC address if avoidable.
	if remote.Sig.Host.Equal(addr.SvcSIG) {
		old := sm.sess.Remote()
		// If the previous remote is not set, do not set the snapshot.
		if old == nil {
			return
		}
		remote.Sig = old.Sig
	}
	sm.sess.currRemote.Store(&remote)
}

func (sm *sessMonitor) getNewPath(old *egress.SessPath) *egress.SessPath {
	if old == nil {
		return sm.sessPathPool.Get("")
	}
	return sm.sessPathPool.Get(old.Key())
}

func (sm *sessMonitor) sendReq() {
	if sm.smRemote == nil || sm.smRemote.SessPath == nil {
		return
	}
	sm.updateMsgId = mgmt.MsgIdType(time.Now().UnixNano())
	spld, err := mgmt.NewPld(sm.updateMsgId, mgmt.NewPollReq(sigcmn.MgmtAddr, sm.sess.SessId))
	if err != nil {
		sm.Error("sessMonitor: Error creating SIGCtrl payload", "err", err)
		return
	}
	cpld, err := ctrl.NewPld(spld, nil)
	if err != nil {
		sm.Error("sessMonitor: Error creating Ctrl payload", "err", err)
		return
	}
	scpld, err := cpld.SignedPld(infra.NullSigner)
	if err != nil {
		sm.Error("sessMonitor: Error creating signed Ctrl payload", "err", err)
		return
	}
	raw, err := scpld.PackPld()
	if err != nil {
		sm.Error("sessMonitor: Error packing signed Ctrl payload", "err", err)
		return
	}
	raddr := sm.smRemote.Sig.CtrlSnetAddr()
	raddr.Path = spath.New(sm.smRemote.SessPath.PathEntry().Path.FwdPath)
	if err := raddr.Path.InitOffsets(); err != nil {
		sm.Error("sessMonitor: Error initializing path offsets", "err", err)
	}
	nh, err := sm.smRemote.SessPath.PathEntry().HostInfo.Overlay()
	if err != nil {
		sm.Error("sessMonitor: Unsupported NextHop", "err", err)
	}
	raddr.NextHop = nh
	// XXX(kormat): if this blocks, both the sessMon and egress worker
	// goroutines will block. Can't just use SetWriteDeadline, as both
	// goroutines write to it.
	_, err = sm.sess.conn.WriteToSCION(raw, raddr)
	if err != nil {
		sm.Error("sessMonitor: Error sending signed Ctrl payload", "err", err)
	}
}

func (sm *sessMonitor) handleRep(rpld *disp.RegPld) {
	pollRep, ok := rpld.P.(*mgmt.PollRep)
	if !ok {
		sm.Error("sessMonitor: non-SIGPollRep payload received",
			"src", rpld.Addr, "type", common.TypeOf(rpld.P), "pld", rpld.P)
		return
	}
	if !sm.sess.IA().Equal(rpld.Addr.IA) {
		sm.Error("sessMonitor: SIGPollRep from wrong IA",
			"expected", sm.sess.IA(), "actual", rpld.Addr.IA)
		return
	}
	// Only update the session's RemoteInfo if we get a response matching
	// the last poll we sent.
	if sm.updateMsgId == rpld.Id {
		sm.lastReply = time.Now()
		// Update sessmon's remote.
		sm.smRemote.Sig = &siginfo.Sig{
			IA:          sm.smRemote.Sig.IA,
			Host:        pollRep.Addr.Ctrl.Host(),
			CtrlL4Port:  int(pollRep.Addr.Ctrl.Port),
			EncapL4Port: int(pollRep.Addr.EncapPort),
		}
		// Update session's remote, if needed.
		sessRemote := sm.sess.Remote()
		if sessRemote == nil || !sm.smRemote.Sig.Equal(sessRemote.Sig) {
			sm.updateSessSnap()
			sm.Info("sessMonitor: updating remote Info", "msgId", rpld.Id, "remote", sm.smRemote)
		}
		sm.sess.healthy.Store(true)
	} else {
		// This is going to happen if latency of the path is greater than the poll ticker period.
		// TODO(sustrik): We should monitor this to spot paths where the latency is high enough to
		// to disrupt orderly SIG operation.
		sm.Info("Reply to an old request received", "request", sm.updateMsgId, "reply", rpld.Id)
	}

}
<|endoftext|>


<commit_before>package fetcher

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"net/url"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	"github.com/syou6162/GoOse"
)

type Article struct {
	Url           string
	Title         string
	Description   string
	OgDescription string
	OgType        string
	OgImage       string
	Body          string
	StatusCode    int
	Favicon       string
}

var articleFetcher = http.Client{
	Transport: &http.Transport{
		MaxIdleConns:        0,
		MaxIdleConnsPerHost: 100,
	},
	Timeout: time.Duration(5 * time.Second),
}

func updateMetaDescriptionIfArxiv(article *goose.Article, origUrl string, finalUrl string, html []byte) error {
	arxivUrl := "https://arxiv.org/abs/"
	if strings.Contains(origUrl, arxivUrl) || strings.Contains(finalUrl, arxivUrl) {
		// article.Docでもいけそうだが、gooseが中で書き換えていてダメ。Documentを作りなおす
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(html)))
		if err != nil {
			return err
		}
		article.MetaDescription = doc.Find(".abstract").Text()
	}
	return nil
}

func GetArticle(origUrl string) (*Article, error) {
	g := goose.New()
	resp, err := articleFetcher.Get(origUrl)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == http.StatusUnauthorized ||
		resp.StatusCode == http.StatusForbidden ||
		resp.StatusCode == http.StatusNotFound ||
		resp.StatusCode == http.StatusGone ||
		resp.StatusCode == http.StatusServiceUnavailable {
		return nil, errors.New(fmt.Sprintf("%s: Cannot fetch %s", resp.Status, origUrl))
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if !utf8.Valid(html) {
		return nil, errors.New(fmt.Sprintf("Invalid utf8 document: %s", origUrl))
	}

	article, err := g.ExtractFromRawHTML(resp.Request.URL.String(), string(html))
	if err != nil {
		return nil, err
	}

	finalUrl := article.CanonicalLink
	if finalUrl == "" {
		finalUrl = resp.Request.URL.String()
	}

	updateMetaDescriptionIfArxiv(article, origUrl, finalUrl, html)

	favicon := ""
	if u, err := url.Parse(article.MetaFavicon); err == nil {
		if u.IsAbs() {
			favicon = article.MetaFavicon
		}
	}

	return &Article{
		Url:           finalUrl,
		Title:         article.Title,
		Description:   article.MetaDescription,
		OgDescription: article.MetaOgDescription,
		OgType:        article.MetaOgType,
		OgImage:       article.MetaOgImage,
		Body:          article.CleanedText,
		StatusCode:    resp.StatusCode,
		Favicon:       favicon,
	}, nil
}
<commit_msg>さらに追加<commit_after>package fetcher

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"net/url"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	"github.com/syou6162/GoOse"
)

type Article struct {
	Url           string
	Title         string
	Description   string
	OgDescription string
	OgType        string
	OgImage       string
	Body          string
	StatusCode    int
	Favicon       string
}

var articleFetcher = http.Client{
	Transport: &http.Transport{
		MaxIdleConns:        0,
		MaxIdleConnsPerHost: 100,
	},
	Timeout: time.Duration(5 * time.Second),
}

func updateMetaDescriptionIfArxiv(article *goose.Article, origUrl string, finalUrl string, html []byte) error {
	arxivUrl := "https://arxiv.org/abs/"
	if strings.Contains(origUrl, arxivUrl) || strings.Contains(finalUrl, arxivUrl) {
		// article.Docでもいけそうだが、gooseが中で書き換えていてダメ。Documentを作りなおす
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(html)))
		if err != nil {
			return err
		}
		article.MetaDescription = doc.Find(".abstract").Text()
	}
	return nil
}

func GetArticle(origUrl string) (*Article, error) {
	g := goose.New()
	resp, err := articleFetcher.Get(origUrl)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == http.StatusFound ||
		resp.StatusCode == http.StatusUnauthorized ||
		resp.StatusCode == http.StatusForbidden ||
		resp.StatusCode == http.StatusNotFound ||
		resp.StatusCode == http.StatusGone ||
		resp.StatusCode == http.StatusBadGateway ||
		resp.StatusCode == http.StatusServiceUnavailable {
		return nil, errors.New(fmt.Sprintf("%s: Cannot fetch %s", resp.Status, origUrl))
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if !utf8.Valid(html) {
		return nil, errors.New(fmt.Sprintf("Invalid utf8 document: %s", origUrl))
	}

	article, err := g.ExtractFromRawHTML(resp.Request.URL.String(), string(html))
	if err != nil {
		return nil, err
	}

	finalUrl := article.CanonicalLink
	if finalUrl == "" {
		finalUrl = resp.Request.URL.String()
	}

	updateMetaDescriptionIfArxiv(article, origUrl, finalUrl, html)

	favicon := ""
	if u, err := url.Parse(article.MetaFavicon); err == nil {
		if u.IsAbs() {
			favicon = article.MetaFavicon
		}
	}

	return &Article{
		Url:           finalUrl,
		Title:         article.Title,
		Description:   article.MetaDescription,
		OgDescription: article.MetaOgDescription,
		OgType:        article.MetaOgType,
		OgImage:       article.MetaOgImage,
		Body:          article.CleanedText,
		StatusCode:    resp.StatusCode,
		Favicon:       favicon,
	}, nil
}
<|endoftext|>


<commit_before>package singularity

import (
	"fmt"

	"github.com/opentable/go-singularity/dtos"
	"github.com/opentable/sous/ext/docker"
	"github.com/opentable/sous/lib"
	"github.com/opentable/sous/util/firsterr"
	"github.com/pkg/errors"
)

type (
	deploymentBuilder struct {
		clusters  sous.Clusters
		Target    sous.DeployState
		imageName string
		depMarker sDepMarker
		history   sHistory
		deploy    sDeploy
		request   sRequest
		req       SingReq
		registry  sous.ImageLabeller
	}

	canRetryRequest struct {
		cause error
		req   SingReq
	}

	malformedResponse struct {
		message string
	}
)

func (mr malformedResponse) Error() string {
	return mr.message
}

func isMalformed(err error) bool {
	err = errors.Cause(err)
	_, yes := err.(malformedResponse)
	Log.Vomit.Printf("err: %+v %T %t", err, err, yes)
	return yes
}

func (cr *canRetryRequest) Error() string {
	return fmt.Sprintf("%s: %s", cr.cause, cr.name())
}

func (cr *canRetryRequest) name() string {
	return fmt.Sprintf("%s:%s", cr.req.SourceURL, cr.req.ReqParent.Request.Id)
}

func (db *deploymentBuilder) canRetry(err error) error {
	if err == nil || !db.isRetryable(err) {
		return err
	}
	return &canRetryRequest{err, db.req}
}

func (db *deploymentBuilder) isRetryable(err error) bool {
	return !isMalformed(err) &&
		db.req.SourceURL != "" &&
		db.req.ReqParent != nil &&
		db.req.ReqParent.Request != nil &&
		db.req.ReqParent.Request.Id != ""
}

// BuildDeployment does all the work to collect the data for a Deployment
// from Singularity based on the initial SingularityRequest.
func BuildDeployment(reg sous.ImageLabeller, clusters sous.Clusters, req SingReq) (sous.DeployState, error) {
	Log.Vomit.Printf("%#v", req.ReqParent)
	db := deploymentBuilder{registry: reg, clusters: clusters, req: req}

	db.Target.Cluster = &sous.Cluster{BaseURL: req.SourceURL}
	db.request = req.ReqParent.Request

	return db.Target, db.canRetry(db.completeConstruction())
}

func (db *deploymentBuilder) completeConstruction() error {
	return firsterr.Returned(
		db.determineDeployStatus,
		db.retrieveDeploy,
		db.extractDeployFromDeployHistory,
		db.determineStatus,
		db.extractArtifactName,
		db.retrieveImageLabels,
		db.assignClusterName,
		db.unpackDeployConfig,
		db.determineManifestKind,
	)
}

func reqID(rp *dtos.SingularityRequestParent) (ID string) {
	defer func() {
		if e := recover(); e != nil {
			return
		}
	}()
	ID = "<null RP>"
	if rp != nil {
		ID = "<null Request>"
	}
	ID = rp.Request.Id
	return
}

// If there is a Pending deploy, as far as Sous is concerned, that's "to
// come" - we optimistically assume it will become Active, and that's the
// Deployment we should consider live.
//
// (At some point in the future we may want to be able to report the "live"
// deployment - at best based on this we could infer that a previous GDM
// entry was running. (consider several quick updates, though...(but
// Singularity semantics mean that each of them that was actually resolved
// would have been Active however briefly (but Sous would accept GDM updates
// arbitrarily quickly as compared to resolve completions...))))
func (db *deploymentBuilder) determineDeployStatus() error {
	logFDs("before retrieveDeploy")
	defer logFDs("after retrieveDeploy")

	rp := db.req.ReqParent
	if rp == nil {
		return malformedResponse{fmt.Sprintf("Singularity response didn't include a request parent. %v", db.req)}
	}

	rds := rp.RequestDeployState

	if rds == nil {
		return malformedResponse{"Singularity response didn't include a deploy state. ReqId: " + reqID(rp)}
	}

	if rds.PendingDeploy != nil {
		db.Target.Status = sous.DeployStatusPending
		db.depMarker = rds.PendingDeploy
	}
	// if there's no Pending deploy, we'll use the top of history in preference to Active
	// Consider: we might collect both and compare timestamps, but the active is
	// going to be the top of the history anyway unless there's been a more
	// recent failed deploy
	return nil
}

func (db *deploymentBuilder) retrieveDeploy() error {
	if db.depMarker == nil {
		return db.retrieveHistoricDeploy()
	}
	Log.Vomit.Printf("Getting deploy based on Pending marker.")
	return db.retrieveLiveDeploy()
}

func (db *deploymentBuilder) retrieveHistoricDeploy() error {
	Log.Vomit.Printf("Getting deploy from history")
	// !!! makes HTTP req
	if db.request == nil {
		return malformedResponse{"Singularity request parent had no request."}
	}
	sing := db.req.Sing
	depHistList, err := sing.GetDeploys(db.request.Id, 1, 1)
	Log.Vomit.Printf("Got history from Singularity with %d items.", len(depHistList))
	if err != nil {
		return errors.Wrap(err, "GetDeploys")
	}

	if len(depHistList) == 0 {
		return malformedResponse{"Singularity deploy history list was empty."}
	}

	partialHistory := depHistList[0]

	Log.Vomit.Printf("%#v", partialHistory)
	if partialHistory.DeployMarker == nil {
		return malformedResponse{"Singularity deploy history had no deploy marker."}
	}

	Log.Vomit.Printf("%#v", partialHistory.DeployMarker)
	db.depMarker = partialHistory.DeployMarker
	db.retrieveLiveDeploy()
	return nil
}

func (db *deploymentBuilder) retrieveLiveDeploy() error {
	// !!! makes HTTP req
	sing := db.req.Sing
	dh, err := sing.GetDeploy(db.depMarker.RequestId, db.depMarker.DeployId)
	if err != nil {
		return errors.Wrapf(err, "%#v", db.depMarker)
	}
	Log.Vomit.Printf("Deploy history entry retrieved: %#v", dh)

	db.history = dh

	return nil
}

func (db *deploymentBuilder) extractDeployFromDeployHistory() error {
	db.deploy = db.history.Deploy
	if db.deploy == nil {
		return malformedResponse{"Singularity deploy history included no deploy"}
	}

	return nil
}

func (db *deploymentBuilder) determineStatus() error {
	if db.history.DeployResult == nil {
		db.Target.Status = sous.DeployStatusPending
		return nil
	}
	if db.history.DeployResult.DeployState == dtos.SingularityDeployResultDeployStateSUCCEEDED {
		db.Target.Status = sous.DeployStatusActive
	} else {
		db.Target.Status = sous.DeployStatusFailed
	}

	return nil
}

func (db *deploymentBuilder) extractArtifactName() error {
	logFDs("before retrieveImageLabels")
	defer logFDs("after retrieveImageLabels")
	ci := db.deploy.ContainerInfo
	if ci == nil {
		return malformedResponse{"Blank container info"}
	}

	if ci.Type != dtos.SingularityContainerInfoSingularityContainerTypeDOCKER {
		return malformedResponse{"Singularity container isn't a docker container"}
	}
	dkr := ci.Docker
	if dkr == nil {
		return malformedResponse{"Singularity deploy didn't include a docker info"}
	}

	db.imageName = dkr.Image
	return nil
}

func (db *deploymentBuilder) retrieveImageLabels() error {
	// XXX coupled to Docker registry as ImageMapper
	// !!! HTTP request
	labels, err := db.registry.ImageLabels(db.imageName)
	if err != nil {
		return malformedResponse{err.Error()}
	}
	Log.Vomit.Print("Labels: ", labels)

	db.Target.SourceID, err = docker.SourceIDFromLabels(labels)
	if err != nil {
		return errors.Wrapf(malformedResponse{err.Error()}, "For reqID: %s", reqID(db.req.ReqParent))
	}

	return nil
}

func (db *deploymentBuilder) assignClusterName() error {
	var posNick string
	matchCount := 0
	for nn, url := range db.clusters {
		url := url.BaseURL
		if url != db.req.SourceURL {
			continue
		}
		posNick = nn
		matchCount++

		id := db.Target.ID()
		id.Cluster = nn

		checkID := MakeRequestID(id)
		sous.Log.Vomit.Printf("Trying hypothetical request ID: %s", checkID)
		if checkID == db.request.Id {
			db.Target.ClusterName = nn
			sous.Log.Debug.Printf("Found cluster: %s", nn)
			break
		}
	}
	if db.Target.ClusterName == "" {
		if matchCount == 1 {
			sous.Log.Debug.Printf("No request ID matched, using first plausible cluster: %s", posNick)
			db.Target.ClusterName = posNick
			return nil
		}
		sous.Log.Debug.Printf("No cluster nickname (%#v) matched request id %s for %s", db.clusters, db.request.Id, db.imageName)
		return malformedResponse{fmt.Sprintf("No cluster nickname (%#v) matched request id %s", db.clusters, db.request.Id)}
	}

	return nil
}

func (db *deploymentBuilder) unpackDeployConfig() error {
	db.Target.Env = db.deploy.Env
	Log.Vomit.Printf("Env: %+v", db.deploy.Env)
	if db.Target.Env == nil {
		db.Target.Env = make(map[string]string)
	}

	singRez := db.deploy.Resources
	if singRez == nil {
		return malformedResponse{"Deploy object lacks resources field"}
	}
	db.Target.Resources = make(sous.Resources)
	db.Target.Resources["cpus"] = fmt.Sprintf("%f", singRez.Cpus)
	db.Target.Resources["memory"] = fmt.Sprintf("%f", singRez.MemoryMb)
	db.Target.Resources["ports"] = fmt.Sprintf("%d", singRez.NumPorts)

	db.Target.NumInstances = int(db.request.Instances)
	db.Target.Owners = make(sous.OwnerSet)
	for _, o := range db.request.Owners {
		db.Target.Owners.Add(o)
	}

	for _, v := range db.deploy.ContainerInfo.Volumes {
		db.Target.DeployConfig.Volumes = append(db.Target.DeployConfig.Volumes,
			&sous.Volume{
				Host:      v.HostPath,
				Container: v.ContainerPath,
				Mode:      sous.VolumeMode(v.Mode),
			})
	}
	Log.Vomit.Printf("Volumes %+v", db.Target.DeployConfig.Volumes)
	if len(db.Target.DeployConfig.Volumes) > 0 {
		Log.Debug.Printf("%+v", db.Target.DeployConfig.Volumes[0])
	}

	return nil
}

func (db *deploymentBuilder) determineManifestKind() error {
	switch db.request.RequestType {
	default:
		return fmt.Errorf("Unrecognized request type returned by Singularity: %v", db.request.RequestType)
	case dtos.SingularityRequestRequestTypeSERVICE:
		db.Target.Kind = sous.ManifestKindService
	case dtos.SingularityRequestRequestTypeWORKER:
		db.Target.Kind = sous.ManifestKindWorker
	case dtos.SingularityRequestRequestTypeON_DEMAND:
		db.Target.Kind = sous.ManifestKindOnDemand
	case dtos.SingularityRequestRequestTypeSCHEDULED:
		db.Target.Kind = sous.ManifestKindScheduled
	case dtos.SingularityRequestRequestTypeRUN_ONCE:
		db.Target.Kind = sous.ManifestKindOnce
	}
	return nil
}
<commit_msg>Logging fixes.<commit_after>package singularity

import (
	"fmt"

	"github.com/opentable/go-singularity/dtos"
	"github.com/opentable/sous/ext/docker"
	"github.com/opentable/sous/lib"
	"github.com/opentable/sous/util/firsterr"
	"github.com/pkg/errors"
)

type (
	deploymentBuilder struct {
		clusters  sous.Clusters
		Target    sous.DeployState
		imageName string
		depMarker sDepMarker
		history   sHistory
		deploy    sDeploy
		request   sRequest
		req       SingReq
		registry  sous.ImageLabeller
	}

	canRetryRequest struct {
		cause error
		req   SingReq
	}

	malformedResponse struct {
		message string
	}
)

func (mr malformedResponse) Error() string {
	return mr.message
}

func isMalformed(err error) bool {
	err = errors.Cause(err)
	_, yes := err.(malformedResponse)
	Log.Vomit.Printf("err: %+v %T %t", err, err, yes)
	return yes
}

func (cr *canRetryRequest) Error() string {
	return fmt.Sprintf("%s: %s", cr.cause, cr.name())
}

func (cr *canRetryRequest) name() string {
	return fmt.Sprintf("%s:%s", cr.req.SourceURL, cr.req.ReqParent.Request.Id)
}

func (db *deploymentBuilder) canRetry(err error) error {
	if err == nil || !db.isRetryable(err) {
		return err
	}
	return &canRetryRequest{err, db.req}
}

func (db *deploymentBuilder) isRetryable(err error) bool {
	return !isMalformed(err) &&
		db.req.SourceURL != "" &&
		db.req.ReqParent != nil &&
		db.req.ReqParent.Request != nil &&
		db.req.ReqParent.Request.Id != ""
}

// BuildDeployment does all the work to collect the data for a Deployment
// from Singularity based on the initial SingularityRequest.
func BuildDeployment(reg sous.ImageLabeller, clusters sous.Clusters, req SingReq) (sous.DeployState, error) {
	Log.Vomit.Printf("%#v", req.ReqParent)
	db := deploymentBuilder{registry: reg, clusters: clusters, req: req}

	db.Target.Cluster = &sous.Cluster{BaseURL: req.SourceURL}
	db.request = req.ReqParent.Request

	return db.Target, db.canRetry(db.completeConstruction())
}

func (db *deploymentBuilder) completeConstruction() error {
	return firsterr.Returned(
		db.determineDeployStatus,
		db.retrieveDeploy,
		db.extractDeployFromDeployHistory,
		db.determineStatus,
		db.extractArtifactName,
		db.retrieveImageLabels,
		db.assignClusterName,
		db.unpackDeployConfig,
		db.determineManifestKind,
	)
}

func reqID(rp *dtos.SingularityRequestParent) (ID string) {
	defer func() {
		if e := recover(); e != nil {
			return
		}
	}()
	ID = "<null RP>"
	if rp != nil {
		ID = "<null Request>"
	}
	ID = rp.Request.Id
	return
}

// If there is a Pending deploy, as far as Sous is concerned, that's "to
// come" - we optimistically assume it will become Active, and that's the
// Deployment we should consider live.
//
// (At some point in the future we may want to be able to report the "live"
// deployment - at best based on this we could infer that a previous GDM
// entry was running. (consider several quick updates, though...(but
// Singularity semantics mean that each of them that was actually resolved
// would have been Active however briefly (but Sous would accept GDM updates
// arbitrarily quickly as compared to resolve completions...))))
func (db *deploymentBuilder) determineDeployStatus() error {
	logFDs("before determineDeployStatus()")
	defer logFDs("after determineDeployStatus()")

	rp := db.req.ReqParent
	if rp == nil {
		return malformedResponse{fmt.Sprintf("Singularity response didn't include a request parent. %v", db.req)}
	}

	rds := rp.RequestDeployState

	if rds == nil {
		return malformedResponse{"Singularity response didn't include a deploy state. ReqId: " + reqID(rp)}
	}

	if rds.PendingDeploy != nil {
		db.Target.Status = sous.DeployStatusPending
		db.depMarker = rds.PendingDeploy
	}
	// if there's no Pending deploy, we'll use the top of history in preference to Active
	// Consider: we might collect both and compare timestamps, but the active is
	// going to be the top of the history anyway unless there's been a more
	// recent failed deploy
	return nil
}

func (db *deploymentBuilder) retrieveDeploy() error {
	if db.depMarker == nil {
		return db.retrieveHistoricDeploy()
	}
	Log.Vomit.Printf("Getting deploy based on Pending marker.")
	return db.retrieveLiveDeploy()
}

func (db *deploymentBuilder) retrieveHistoricDeploy() error {
	Log.Vomit.Printf("Getting deploy from history")
	// !!! makes HTTP req
	if db.request == nil {
		return malformedResponse{"Singularity request parent had no request."}
	}
	sing := db.req.Sing
	depHistList, err := sing.GetDeploys(db.request.Id, 1, 1)
	Log.Vomit.Printf("Got history from Singularity with %d items.", len(depHistList))
	if err != nil {
		return errors.Wrap(err, "GetDeploys")
	}

	if len(depHistList) == 0 {
		return malformedResponse{"Singularity deploy history list was empty."}
	}

	partialHistory := depHistList[0]

	Log.Vomit.Printf("%#v", partialHistory)
	if partialHistory.DeployMarker == nil {
		return malformedResponse{"Singularity deploy history had no deploy marker."}
	}

	Log.Vomit.Printf("%#v", partialHistory.DeployMarker)
	db.depMarker = partialHistory.DeployMarker
	db.retrieveLiveDeploy()
	return nil
}

func (db *deploymentBuilder) retrieveLiveDeploy() error {
	// !!! makes HTTP req
	sing := db.req.Sing
	dh, err := sing.GetDeploy(db.depMarker.RequestId, db.depMarker.DeployId)
	if err != nil {
		return errors.Wrapf(err, "%#v", db.depMarker)
	}
	Log.Vomit.Printf("Deploy history entry retrieved: %#v", dh)

	db.history = dh

	return nil
}

func (db *deploymentBuilder) extractDeployFromDeployHistory() error {
	db.deploy = db.history.Deploy
	if db.deploy == nil {
		return malformedResponse{"Singularity deploy history included no deploy"}
	}

	return nil
}

func (db *deploymentBuilder) determineStatus() error {
	if db.history.DeployResult == nil {
		db.Target.Status = sous.DeployStatusPending
		return nil
	}
	if db.history.DeployResult.DeployState == dtos.SingularityDeployResultDeployStateSUCCEEDED {
		db.Target.Status = sous.DeployStatusActive
	} else {
		db.Target.Status = sous.DeployStatusFailed
	}

	return nil
}

func (db *deploymentBuilder) extractArtifactName() error {
	logFDs("before extractArtifactName()")
	defer logFDs("after extractArtifactName()")
	ci := db.deploy.ContainerInfo
	if ci == nil {
		return malformedResponse{"Blank container info"}
	}

	if ci.Type != dtos.SingularityContainerInfoSingularityContainerTypeDOCKER {
		return malformedResponse{"Singularity container isn't a docker container"}
	}
	dkr := ci.Docker
	if dkr == nil {
		return malformedResponse{"Singularity deploy didn't include a docker info"}
	}

	db.imageName = dkr.Image
	return nil
}

func (db *deploymentBuilder) retrieveImageLabels() error {
	// XXX coupled to Docker registry as ImageMapper
	// !!! HTTP request
	labels, err := db.registry.ImageLabels(db.imageName)
	if err != nil {
		return malformedResponse{err.Error()}
	}
	Log.Vomit.Print("Labels: ", labels)

	db.Target.SourceID, err = docker.SourceIDFromLabels(labels)
	if err != nil {
		return errors.Wrapf(malformedResponse{err.Error()}, "For reqID: %s", reqID(db.req.ReqParent))
	}

	return nil
}

func (db *deploymentBuilder) assignClusterName() error {
	var posNick string
	matchCount := 0
	for nn, url := range db.clusters {
		url := url.BaseURL
		if url != db.req.SourceURL {
			continue
		}
		posNick = nn
		matchCount++

		id := db.Target.ID()
		id.Cluster = nn

		checkID := MakeRequestID(id)
		sous.Log.Vomit.Printf("Trying hypothetical request ID: %s", checkID)
		if checkID == db.request.Id {
			db.Target.ClusterName = nn
			sous.Log.Debug.Printf("Found cluster: %s", nn)
			break
		}
	}
	if db.Target.ClusterName == "" {
		if matchCount == 1 {
			sous.Log.Debug.Printf("No request ID matched, using first plausible cluster: %s", posNick)
			db.Target.ClusterName = posNick
			return nil
		}
		sous.Log.Debug.Printf("No cluster nickname (%#v) matched request id %s for %s", db.clusters, db.request.Id, db.imageName)
		return malformedResponse{fmt.Sprintf("No cluster nickname (%#v) matched request id %s", db.clusters, db.request.Id)}
	}

	return nil
}

func (db *deploymentBuilder) unpackDeployConfig() error {
	db.Target.Env = db.deploy.Env
	Log.Vomit.Printf("Env: %+v", db.deploy.Env)
	if db.Target.Env == nil {
		db.Target.Env = make(map[string]string)
	}

	singRez := db.deploy.Resources
	if singRez == nil {
		return malformedResponse{"Deploy object lacks resources field"}
	}
	db.Target.Resources = make(sous.Resources)
	db.Target.Resources["cpus"] = fmt.Sprintf("%f", singRez.Cpus)
	db.Target.Resources["memory"] = fmt.Sprintf("%f", singRez.MemoryMb)
	db.Target.Resources["ports"] = fmt.Sprintf("%d", singRez.NumPorts)

	db.Target.NumInstances = int(db.request.Instances)
	db.Target.Owners = make(sous.OwnerSet)
	for _, o := range db.request.Owners {
		db.Target.Owners.Add(o)
	}

	for _, v := range db.deploy.ContainerInfo.Volumes {
		db.Target.DeployConfig.Volumes = append(db.Target.DeployConfig.Volumes,
			&sous.Volume{
				Host:      v.HostPath,
				Container: v.ContainerPath,
				Mode:      sous.VolumeMode(v.Mode),
			})
	}
	Log.Vomit.Printf("Volumes %+v", db.Target.DeployConfig.Volumes)
	if len(db.Target.DeployConfig.Volumes) > 0 {
		Log.Debug.Printf("%+v", db.Target.DeployConfig.Volumes[0])
	}

	return nil
}

func (db *deploymentBuilder) determineManifestKind() error {
	switch db.request.RequestType {
	default:
		return fmt.Errorf("Unrecognized request type returned by Singularity: %v", db.request.RequestType)
	case dtos.SingularityRequestRequestTypeSERVICE:
		db.Target.Kind = sous.ManifestKindService
	case dtos.SingularityRequestRequestTypeWORKER:
		db.Target.Kind = sous.ManifestKindWorker
	case dtos.SingularityRequestRequestTypeON_DEMAND:
		db.Target.Kind = sous.ManifestKindOnDemand
	case dtos.SingularityRequestRequestTypeSCHEDULED:
		db.Target.Kind = sous.ManifestKindScheduled
	case dtos.SingularityRequestRequestTypeRUN_ONCE:
		db.Target.Kind = sous.ManifestKindOnce
	}
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSCallerIdentity_basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsCallerIdentityConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsCallerIdentityAccountId("data.aws_caller_identity.current"),
				),
			},
		},
	})
}

func testAccCheckAwsCallerIdentityAccountId(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find AccountID resource: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("Account Id resource ID not set.")
		}

		expected := testAccProvider.Meta().(*AWSClient).accountid
		if rs.Primary.Attributes["account_id"] != expected {
			return fmt.Errorf("Incorrect Account ID: expected %q, got %q", expected, rs.Primary.ID)
		}

		return nil
	}
}

const testAccCheckAwsCallerIdentityConfig_basic = `
data "aws_caller_identity" "current" { }
`
<commit_msg>Correct error message for 'aws_caller_identity' acceptance test. (#10712)<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSCallerIdentity_basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsCallerIdentityConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsCallerIdentityAccountId("data.aws_caller_identity.current"),
				),
			},
		},
	})
}

func testAccCheckAwsCallerIdentityAccountId(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find AccountID resource: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("Account Id resource ID not set.")
		}

		expected := testAccProvider.Meta().(*AWSClient).accountid
		if rs.Primary.Attributes["account_id"] != expected {
			return fmt.Errorf("Incorrect Account ID: expected %q, got %q", expected, rs.Primary.Attributes["account_id"])
		}

		return nil
	}
}

const testAccCheckAwsCallerIdentityConfig_basic = `
data "aws_caller_identity" "current" { }
`
<|endoftext|>


<commit_before>package mcnutils

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"runtime"
	"strconv"
	"time"
)

type MultiError struct {
	Errs []error
}

func (e MultiError) Error() string {
	aggregate := ""
	for _, err := range e.Errs {
		aggregate += err.Error() + "\n"
	}
	return aggregate
}

// GetHomeDir returns the home directory
// TODO: Having this here just strikes me as dangerous, but some of the drivers
// depend on it ;_;
func GetHomeDir() string {
	if runtime.GOOS == "windows" {
		return os.Getenv("USERPROFILE")
	}
	return os.Getenv("HOME")
}

func GetUsername() string {
	u := "unknown"
	osUser := ""

	switch runtime.GOOS {
	case "darwin", "linux":
		osUser = os.Getenv("USER")
	case "windows":
		osUser = os.Getenv("USERNAME")
	}

	if osUser != "" {
		u = osUser
	}

	return u
}

func CopyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}

	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return err
	}

	defer out.Close()

	if _, err = io.Copy(out, in); err != nil {
		return err
	}

	fi, err := os.Stat(src)
	if err != nil {
		return err
	}

	return os.Chmod(dst, fi.Mode())
}

func WaitForSpecificOrError(f func() (bool, error), maxAttempts int, waitInterval time.Duration) error {
	for i := 0; i < maxAttempts; i++ {
		stop, err := f()
		if err != nil {
			return err
		}
		if stop {
			return nil
		}
		time.Sleep(waitInterval)
	}
	return fmt.Errorf("Maximum number of retries (%d) exceeded", maxAttempts)
}

func WaitForSpecific(f func() bool, maxAttempts int, waitInterval time.Duration) error {
	return WaitForSpecificOrError(func() (bool, error) {
		return f(), nil
	}, maxAttempts, waitInterval)
}

func WaitFor(f func() bool) error {
	i, err := strconv.Atoi(os.Getenv("MAX_ATTEMPTS"))
    	if err != nil {
                i=60
    	}
    	return WaitForSpecific(f, i, 3*time.Second)

}

// TruncateID returns a shorten id
// Following two functions are from github.com/docker/docker/utils module. It
// was way overkill to include the whole module, so we just have these bits
// that we're using here.
func TruncateID(id string) string {
	shortLen := 12
	if len(id) < shortLen {
		shortLen = len(id)
	}
	return id[:shortLen]
}

// GenerateRandomID returns an unique id
func GenerateRandomID() string {
	for {
		id := make([]byte, 32)
		if _, err := io.ReadFull(rand.Reader, id); err != nil {
			panic(err) // This shouldn't happen
		}
		value := hex.EncodeToString(id)
		// if we try to parse the truncated for as an int and we don't have
		// an error then the value is all numeric and causes issues when
		// used as a hostname. ref #3869
		if _, err := strconv.ParseInt(TruncateID(value), 10, 64); err == nil {
			continue
		}
		return value
	}
}
<commit_msg>correct indentation<commit_after>package mcnutils

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"runtime"
	"strconv"
	"time"
)

type MultiError struct {
	Errs []error
}

func (e MultiError) Error() string {
	aggregate := ""
	for _, err := range e.Errs {
		aggregate += err.Error() + "\n"
	}
	return aggregate
}

// GetHomeDir returns the home directory
// TODO: Having this here just strikes me as dangerous, but some of the drivers
// depend on it ;_;
func GetHomeDir() string {
	if runtime.GOOS == "windows" {
		return os.Getenv("USERPROFILE")
	}
	return os.Getenv("HOME")
}

func GetUsername() string {
	u := "unknown"
	osUser := ""

	switch runtime.GOOS {
	case "darwin", "linux":
		osUser = os.Getenv("USER")
	case "windows":
		osUser = os.Getenv("USERNAME")
	}

	if osUser != "" {
		u = osUser
	}

	return u
}

func CopyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}

	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return err
	}

	defer out.Close()

	if _, err = io.Copy(out, in); err != nil {
		return err
	}

	fi, err := os.Stat(src)
	if err != nil {
		return err
	}

	return os.Chmod(dst, fi.Mode())
}

func WaitForSpecificOrError(f func() (bool, error), maxAttempts int, waitInterval time.Duration) error {
	for i := 0; i < maxAttempts; i++ {
		stop, err := f()
		if err != nil {
			return err
		}
		if stop {
			return nil
		}
		time.Sleep(waitInterval)
	}
	return fmt.Errorf("Maximum number of retries (%d) exceeded", maxAttempts)
}

func WaitForSpecific(f func() bool, maxAttempts int, waitInterval time.Duration) error {
	return WaitForSpecificOrError(func() (bool, error) {
		return f(), nil
	}, maxAttempts, waitInterval)
}

func WaitFor(f func() bool) error {
	i, err := strconv.Atoi(os.Getenv("MAX_ATTEMPTS"))
	if err != nil {
		i = 60
	}
	return WaitForSpecific(f, i, 3*time.Second)
}

// TruncateID returns a shorten id
// Following two functions are from github.com/docker/docker/utils module. It
// was way overkill to include the whole module, so we just have these bits
// that we're using here.
func TruncateID(id string) string {
	shortLen := 12
	if len(id) < shortLen {
		shortLen = len(id)
	}
	return id[:shortLen]
}

// GenerateRandomID returns an unique id
func GenerateRandomID() string {
	for {
		id := make([]byte, 32)
		if _, err := io.ReadFull(rand.Reader, id); err != nil {
			panic(err) // This shouldn't happen
		}
		value := hex.EncodeToString(id)
		// if we try to parse the truncated for as an int and we don't have
		// an error then the value is all numeric and causes issues when
		// used as a hostname. ref #3869
		if _, err := strconv.ParseInt(TruncateID(value), 10, 64); err == nil {
			continue
		}
		return value
	}
}
<|endoftext|>


<commit_before>package httpaccesslog

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type responseStats struct {
	bodyBytes  int
	statusCode int
}

type LogResponseWriter struct {
	http.ResponseWriter
	Stats *responseStats
}

func (this LogResponseWriter) Write(responseBody []byte) (int, error) {
	this.Stats.bodyBytes = len(responseBody)
	return this.ResponseWriter.Write(responseBody)
}

func (this LogResponseWriter) WriteHeader(statusCode int) {
	this.Stats.statusCode = statusCode
	this.ResponseWriter.WriteHeader(statusCode)
}

type AccessLogger struct {
	log.Logger
}

func (this AccessLogger) Handle(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		startTime := time.Now()
		stats := responseStats{0, 200}
		logResponseWriter := LogResponseWriter{w, &stats}
		handler(logResponseWriter, r)
		requestDuration := time.Now().Sub(startTime)
		this.Logger.Println(formatAccessLog(r, time.Now(), stats, requestDuration, requestDuration, 0))
	}
}

func formatAccessLog(
	r *http.Request,
	dateTime time.Time,
	stats responseStats,
	requestTime, upstreamTime time.Duration,
	compressionRatio float64) string {
	username, _, ok := r.BasicAuth()
	if !ok || username == "" {
		username = "-"
	}
	referer := "-"
	userAgent := "-"
	if len(r.Header["Referer"]) > 0 {
		referer = r.Header["Referer"][0]
	}
	if len(r.Header["UserAgent"]) > 0 {
		userAgent = r.Header["UserAgent"][0]
	}

	remoteAddr := strings.Split(r.RemoteAddr, ":")
	compressionRatioStr := "-"
	if compressionRatio > 0 {
		compressionRatioStr = strconv.FormatFloat(compressionRatio, 'f', 2, 64)
	}
	requestSeconds := strconv.FormatFloat(float64(requestTime)/float64(time.Second), 'f', 3, 64)
	upstreamSeconds := strconv.FormatFloat(float64(upstreamTime)/float64(time.Second), 'f', 3, 64)
	return fmt.Sprintf(
		"%s - %s [%s] \"%s %s %s\" %d %d %s/%s \"%s\" \"%s\" %s -",
		remoteAddr[0],
		username,
		dateTime.Format("02/Jan/2006:15:04:05 -0700"),
		r.Method,
		r.URL,
		r.Proto,
		stats.statusCode,
		stats.bodyBytes,
		requestSeconds,
		upstreamSeconds,
		referer,
		userAgent,
		compressionRatioStr,
	)
}
<commit_msg>make some types and members private<commit_after>package httpaccesslog

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type responseStats struct {
	bodyBytes  int
	statusCode int
}

type logResponseWriter struct {
	http.ResponseWriter
	stats *responseStats
}

func (this logResponseWriter) Write(responseBody []byte) (int, error) {
	this.stats.bodyBytes = len(responseBody)
	return this.ResponseWriter.Write(responseBody)
}

func (this logResponseWriter) WriteHeader(statusCode int) {
	this.stats.statusCode = statusCode
	this.ResponseWriter.WriteHeader(statusCode)
}

type AccessLogger struct {
	log.Logger
}

func (this AccessLogger) Handle(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		startTime := time.Now()
		stats := responseStats{0, 200}
		logWriter := logResponseWriter{w, &stats}
		handler(logWriter, r)
		requestDuration := time.Now().Sub(startTime)
		this.Logger.Println(formatAccessLog(r, time.Now(), stats, requestDuration, requestDuration, 0))
	}
}

func formatAccessLog(
	r *http.Request,
	dateTime time.Time,
	stats responseStats,
	requestTime, upstreamTime time.Duration,
	compressionRatio float64) string {
	username, _, ok := r.BasicAuth()
	if !ok || username == "" {
		username = "-"
	}
	referer := "-"
	userAgent := "-"
	if len(r.Header["Referer"]) > 0 {
		referer = r.Header["Referer"][0]
	}
	if len(r.Header["UserAgent"]) > 0 {
		userAgent = r.Header["UserAgent"][0]
	}

	remoteAddr := strings.Split(r.RemoteAddr, ":")
	compressionRatioStr := "-"
	if compressionRatio > 0 {
		compressionRatioStr = strconv.FormatFloat(compressionRatio, 'f', 2, 64)
	}
	requestSeconds := strconv.FormatFloat(float64(requestTime)/float64(time.Second), 'f', 3, 64)
	upstreamSeconds := strconv.FormatFloat(float64(upstreamTime)/float64(time.Second), 'f', 3, 64)
	return fmt.Sprintf(
		"%s - %s [%s] \"%s %s %s\" %d %d %s/%s \"%s\" \"%s\" %s -",
		remoteAddr[0],
		username,
		dateTime.Format("02/Jan/2006:15:04:05 -0700"),
		r.Method,
		r.URL,
		r.Proto,
		stats.statusCode,
		stats.bodyBytes,
		requestSeconds,
		upstreamSeconds,
		referer,
		userAgent,
		compressionRatioStr,
	)
}
<|endoftext|>


<commit_before>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package config

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io"
	"strings"
	"sync"
)

// V2 defines a V2 config. Public fields are accessible by `json`
// encoders and decoder.
//
// On first call to GetPermission* methods, it initializes an internal ACL
// checker. If the object is constructed from ParseConfig, its internal ACL
// checker is initialized automatically. Any changes to the ACL fields
// afterwards have no effect.
type V2 struct {
	Common

	// Users is a [username -> sha256-hashed password in form of
	// "sha256:<hex>"] map that defines how users should be authenticated.
	Users map[string]string `json:"users"`

	// username -> sha256 hash
	users map[string][]byte `json:"users"`

	// ACLs is a path -> AccessControlV1 map that defines ACLs for different
	// paths.
	ACLs map[string]AccessControlV1 `json:"acls"`

	initOnce   sync.Once
	aclChecker *aclCheckerV1
	initErr    error
}

var _ Config = (*V2)(nil)

// DefaultV2 returns a default V2 config, which allows anonymous read to
// everything.
func DefaultV2() *V2 {
	v2 := &V2{
		Common: Common{
			Version: Version2Str,
		},
		ACLs: map[string]AccessControlV1{
			"/": AccessControlV1{
				AnonymousPermissions: "read,list",
			},
		},
	}
	v2.EnsureInit()
	return v2
}

// InvalidPasswordHash is the error that happens when there's an invalid
// password hash in the config.
type InvalidPasswordHash struct{}

// Error implements the error interface.
func (InvalidPasswordHash) Error() string {
	return "invalid passwordhash"
}

const sha256PasswordHashPrefix = "sha256:"

func (c *V2) init() {
	c.aclChecker, c.initErr = makeACLCheckerV1(c.ACLs, c.Users)
	if c.initErr != nil {
		return
	}
	c.users = make(map[string][]byte)
	for username, pass := range c.Users {
		if !strings.HasPrefix(pass, sha256PasswordHashPrefix) {
			c.initErr = InvalidPasswordHash{}
			return
		}
		if len(pass) != hex.EncodedLen(sha256.Size)+len(sha256PasswordHashPrefix) {
			c.initErr = InvalidPasswordHash{}
			return
		}
		c.users[username], c.initErr = hex.DecodeString(pass[len(sha256PasswordHashPrefix):])
		if c.initErr != nil {
			return
		}
	}
}

// EnsureInit initializes c, and returns any error encountered during the
// initialization. It is not necessary to call EnsureInit. Methods that need it
// does it automatically.
func (c *V2) EnsureInit() error {
	c.initOnce.Do(c.init)
	if c.initErr != nil {
		return c.initErr
	}
	return nil
}

// Version implements the Config interface.
func (c *V2) Version() Version {
	return Version2
}

// Authenticate implements the Config interface.
func (c *V2) Authenticate(ctx context.Context, username, cleartextPassword string) bool {
	if c.EnsureInit() != nil {
		return false
	}

	passwordHash, ok := c.users[username]
	if !ok {
		return false
	}

	clearSum := sha256.Sum256([]byte(cleartextPassword))
	return bytes.Equal(clearSum[:], passwordHash)
}

// GetPermissions implements the Config interface.
func (c *V2) GetPermissions(path string, username *string) (
	read, list bool,
	possibleRead, possibleList bool,
	realm string, err error) {
	if err = c.EnsureInit(); err != nil {
		return false, false, false, false, "", err
	}

	perms, maxPerms, realm := c.aclChecker.getPermissions(path, username)
	return perms.read, perms.list, maxPerms.read, maxPerms.list, realm, nil
}

// Encode implements the Config interface.
func (c *V2) Encode(w io.Writer, prettify bool) error {
	encoder := json.NewEncoder(w)
	if prettify {
		encoder.SetIndent("", strings.Repeat(" ", 2))
	}
	return encoder.Encode(c)
}

// Validate checks all public fields of c, and returns an error if any of them
// is invalid, or a nil-error if they are all valid.
//
// Although changes to ACL fields have no effect to ACL checkings once the
// internal ACL checker is intialized (see comment on V2), this method still
// checks the updated ACL feilds. So it's OK to use Validate directly on a
// *V2 that has been modified since it was initialized.
//
// As a result, unlike other methods on the type, this method is not goroutine
// safe against changes to the public fields.
func (c *V2) Validate() error {
	_, err := makeACLCheckerV1(c.ACLs, c.Users)
	return err
}
<commit_msg>fix vet<commit_after>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package config

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io"
	"strings"
	"sync"
)

// V2 defines a V2 config. Public fields are accessible by `json`
// encoders and decoder.
//
// On first call to GetPermission* methods, it initializes an internal ACL
// checker. If the object is constructed from ParseConfig, its internal ACL
// checker is initialized automatically. Any changes to the ACL fields
// afterwards have no effect.
type V2 struct {
	Common

	// Users is a [username -> sha256-hashed password in form of
	// "sha256:<hex>"] map that defines how users should be authenticated.
	Users map[string]string `json:"users"`

	// username -> sha256 hash
	users map[string][]byte

	// ACLs is a path -> AccessControlV1 map that defines ACLs for different
	// paths.
	ACLs map[string]AccessControlV1 `json:"acls"`

	initOnce   sync.Once
	aclChecker *aclCheckerV1
	initErr    error
}

var _ Config = (*V2)(nil)

// DefaultV2 returns a default V2 config, which allows anonymous read to
// everything.
func DefaultV2() *V2 {
	v2 := &V2{
		Common: Common{
			Version: Version2Str,
		},
		ACLs: map[string]AccessControlV1{
			"/": AccessControlV1{
				AnonymousPermissions: "read,list",
			},
		},
	}
	v2.EnsureInit()
	return v2
}

// InvalidPasswordHash is the error that happens when there's an invalid
// password hash in the config.
type InvalidPasswordHash struct{}

// Error implements the error interface.
func (InvalidPasswordHash) Error() string {
	return "invalid passwordhash"
}

const sha256PasswordHashPrefix = "sha256:"

func (c *V2) init() {
	c.aclChecker, c.initErr = makeACLCheckerV1(c.ACLs, c.Users)
	if c.initErr != nil {
		return
	}
	c.users = make(map[string][]byte)
	for username, pass := range c.Users {
		if !strings.HasPrefix(pass, sha256PasswordHashPrefix) {
			c.initErr = InvalidPasswordHash{}
			return
		}
		if len(pass) != hex.EncodedLen(sha256.Size)+len(sha256PasswordHashPrefix) {
			c.initErr = InvalidPasswordHash{}
			return
		}
		c.users[username], c.initErr = hex.DecodeString(pass[len(sha256PasswordHashPrefix):])
		if c.initErr != nil {
			return
		}
	}
}

// EnsureInit initializes c, and returns any error encountered during the
// initialization. It is not necessary to call EnsureInit. Methods that need it
// does it automatically.
func (c *V2) EnsureInit() error {
	c.initOnce.Do(c.init)
	if c.initErr != nil {
		return c.initErr
	}
	return nil
}

// Version implements the Config interface.
func (c *V2) Version() Version {
	return Version2
}

// Authenticate implements the Config interface.
func (c *V2) Authenticate(ctx context.Context, username, cleartextPassword string) bool {
	if c.EnsureInit() != nil {
		return false
	}

	passwordHash, ok := c.users[username]
	if !ok {
		return false
	}

	clearSum := sha256.Sum256([]byte(cleartextPassword))
	return bytes.Equal(clearSum[:], passwordHash)
}

// GetPermissions implements the Config interface.
func (c *V2) GetPermissions(path string, username *string) (
	read, list bool,
	possibleRead, possibleList bool,
	realm string, err error) {
	if err = c.EnsureInit(); err != nil {
		return false, false, false, false, "", err
	}

	perms, maxPerms, realm := c.aclChecker.getPermissions(path, username)
	return perms.read, perms.list, maxPerms.read, maxPerms.list, realm, nil
}

// Encode implements the Config interface.
func (c *V2) Encode(w io.Writer, prettify bool) error {
	encoder := json.NewEncoder(w)
	if prettify {
		encoder.SetIndent("", strings.Repeat(" ", 2))
	}
	return encoder.Encode(c)
}

// Validate checks all public fields of c, and returns an error if any of them
// is invalid, or a nil-error if they are all valid.
//
// Although changes to ACL fields have no effect to ACL checkings once the
// internal ACL checker is intialized (see comment on V2), this method still
// checks the updated ACL feilds. So it's OK to use Validate directly on a
// *V2 that has been modified since it was initialized.
//
// As a result, unlike other methods on the type, this method is not goroutine
// safe against changes to the public fields.
func (c *V2) Validate() error {
	_, err := makeACLCheckerV1(c.ACLs, c.Users)
	return err
}
<|endoftext|>


<commit_before>package game

const (
	// Player species.
	SpecHuman = "human"

	// Monster species.
	SpecOrc = "orc"
)

var (
	PlayerSpec = &Spec{
		Family:  FamActor,
		Genus:   GenPlayer,
		Species: SpecHuman,
		Name:    "DEBO",
		Traits: &Traits{
			Mover:    NewActorMover,
			Fighter:  NewPlayerFighter,
			Packer:   NewActorPacker,
			Equipper: NewActorEquipper,
			Sheet:    NewPlayerSheet,
		},
	}
	MonOrc = &Spec{
		Family:  FamActor,
		Genus:   GenMonster,
		Species: SpecOrc,
		Name:    "ORC",
		Traits: &Traits{
			Mover:   NewActorMover,
			AI:      NewRandomAI,
			Fighter: NewMonsterFighter,
			Packer:  NewActorPacker,
			Sheet: NewMonsterSheet(MonsterSheet{
				str: 2,
				agi: 0,
				vit: 1,
				mnd: 0,

				melee:   1,
				evasion: 1,
				maxhp:   20,
				maxmp:   10,

				damroll:  NewDice(2, 5),
				protroll: NewDice(1, 4),
			}),
		},
	}
)
<commit_msg>Oops.<commit_after>package game

const (
	// Player species.
	SpecHuman = "human"

	// Monster species.
	SpecOrc = "orc"
)

var (
	PlayerSpec = &Spec{
		Family:  FamActor,
		Genus:   GenPlayer,
		Species: SpecHuman,
		Name:    "DEBO",
		Traits: &Traits{
			Mover:    NewActorMover,
			Fighter:  NewActorFighter,
			Packer:   NewActorPacker,
			Equipper: NewActorEquipper,
			Sheet:    NewPlayerSheet,
		},
	}
	MonOrc = &Spec{
		Family:  FamActor,
		Genus:   GenMonster,
		Species: SpecOrc,
		Name:    "ORC",
		Traits: &Traits{
			Mover:   NewActorMover,
			AI:      NewRandomAI,
			Fighter: NewActorFighter,
			Packer:  NewActorPacker,
			Sheet: NewMonsterSheet(MonsterSheet{
				str: 2,
				agi: 0,
				vit: 1,
				mnd: 0,

				melee:   1,
				evasion: 1,
				maxhp:   20,
				maxmp:   10,

				damroll:  NewDice(2, 5),
				protroll: NewDice(1, 4),
			}),
		},
	}
)
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"strings"

	"github.com/getsentry/raven-go"
)

func main() {
	flagCfgFile := flag.String("config", "conf/config.json", "Config file")
	flagSyslog := flag.Bool("syslog", false, "also log to syslog")
	flagLogLevel := flag.String("loglevel", "debug", "debug, info, notice, warning, error")
	flagBots := flag.String(
		"bots", "tg,bgm,pixiv",
		"Bots to start: qw qqWatch, tt twitterTrack, ts twitterSelf, tg telegram, bgm bgmTrack, pixiv pixivFollow")
	flag.Parse()

	var logFlags byte
	if *flagSyslog {
		logFlags = logFlags | LOGSYS
	}
	logger = GetLogger("yubari", *flagLogLevel, logFlags|LOGCOLOR)

	cfg := ReadConfig(flagCfgFile)
	logger.Debugf("Starting with config: %+v", cfg)

	if cfg.SentryDSN != "" {
		raven.SetDSN(cfg.SentryDSN)
		raven.CapturePanic(func() {
			// do all of the scary things here
		}, nil)
	}

	var err error
	redisClient, err = NewRedisClient(cfg.Redis)
	if err != nil {
		logger.Panic(err)
		return
	}
	defer redisClient.Close()
	logger.Debugf("Redis connected: %+v", redisClient)

	// qqBot = NewQQBot(cfg)
	// defer qqBot.Client.Close()
	// logger.Debugf("QQBot: %+v", qqBot)

	twitterBot = NewTwitterBot(cfg.Twitter)
	logger.Debugf("TwitterBot: %+v", twitterBot)

	telegramBot = NewTelegramBot(cfg)
	logger.Debugf("TelegramBot: %+v", telegramBot)

	bots := strings.Split(*flagBots, ",")
	botsLaunched := 0
	for _, b := range bots {
		switch b {
		// case "qw":
		// logger.Debug("Bot: qqWatch")
		// messages := make(chan map[string]string)
		// go qqBot.Poll(messages)
		// go qqWatch(messages)
		// botsLaunched++
		// case "tt":
		// 	logger.Debug("Bot: twitterTrack")
		// 	go twitterBot.Track()
		// 	botsLaunched++
		// case "ts":
		// 	logger.Debug("Bot: twitterSelf")
		// 	go twitterBot.Self()
		// 	botsLaunched++
		case "tg":
			logger.Debug("Bot: telegram")
			go telegramBot.tgBot()
			go telegramBot.delMessage()
			botsLaunched++
		case "bgm":
			logger.Debug("Bot: bgmTrack")
			go bgmTrack(cfg.BgmID, 10)
			botsLaunched++
		case "pixiv":
			logger.Debug("Bot: pixivFollow")
			go pixivFollow(cfg.Pixiv, 20)
			botsLaunched++
		default:
			logger.Warningf("Bot %s is not supported.", b)
		}
	}
	if botsLaunched > 0 {
		select {}
	}
	logger.Notice("Not bots launched.")
}
<commit_msg>renable qw<commit_after>package main

import (
	"flag"
	"strings"

	"github.com/getsentry/raven-go"
)

func main() {
	flagCfgFile := flag.String("config", "conf/config.json", "Config file")
	flagSyslog := flag.Bool("syslog", false, "also log to syslog")
	flagLogLevel := flag.String("loglevel", "debug", "debug, info, notice, warning, error")
	flagBots := flag.String(
		"bots", "tg,bgm,pixiv",
		"Bots to start: qw qqWatch, tt twitterTrack, ts twitterSelf, tg telegram, bgm bgmTrack, pixiv pixivFollow")
	flag.Parse()

	var logFlags byte
	if *flagSyslog {
		logFlags = logFlags | LOGSYS
	}
	logger = GetLogger("yubari", *flagLogLevel, logFlags|LOGCOLOR)

	cfg := ReadConfig(flagCfgFile)
	logger.Debugf("Starting with config: %+v", cfg)

	if cfg.SentryDSN != "" {
		raven.SetDSN(cfg.SentryDSN)
		raven.CapturePanic(func() {
			// do all of the scary things here
		}, nil)
	}

	var err error
	redisClient, err = NewRedisClient(cfg.Redis)
	if err != nil {
		logger.Panic(err)
		return
	}
	defer redisClient.Close()
	logger.Debugf("Redis connected: %+v", redisClient)

	qqBot = NewQQBot(cfg)
	defer qqBot.Client.Close()
	logger.Debugf("QQBot: %+v", qqBot)

	twitterBot = NewTwitterBot(cfg.Twitter)
	logger.Debugf("TwitterBot: %+v", twitterBot)

	telegramBot = NewTelegramBot(cfg)
	logger.Debugf("TelegramBot: %+v", telegramBot)

	bots := strings.Split(*flagBots, ",")
	botsLaunched := 0
	for _, b := range bots {
		switch b {
		case "qw":
			logger.Debug("Bot: qqWatch")
			messages := make(chan map[string]string)
			go qqBot.Poll(messages)
			go qqWatch(messages)
			botsLaunched++
		// case "tt":
		// 	logger.Debug("Bot: twitterTrack")
		// 	go twitterBot.Track()
		// 	botsLaunched++
		// case "ts":
		// 	logger.Debug("Bot: twitterSelf")
		// 	go twitterBot.Self()
		// 	botsLaunched++
		case "tg":
			logger.Debug("Bot: telegram")
			go telegramBot.tgBot()
			go telegramBot.delMessage()
			botsLaunched++
		case "bgm":
			logger.Debug("Bot: bgmTrack")
			go bgmTrack(cfg.BgmID, 10)
			botsLaunched++
		case "pixiv":
			logger.Debug("Bot: pixivFollow")
			go pixivFollow(cfg.Pixiv, 20)
			botsLaunched++
		default:
			logger.Warningf("Bot %s is not supported.", b)
		}
	}
	if botsLaunched > 0 {
		select {}
	}
	logger.Notice("Not bots launched.")
}
<|endoftext|>


<commit_before>package models

import (
	. "eaciit/wfdemo-git/library/helper"
	"time"

	"github.com/eaciit/orm"
	tk "github.com/eaciit/toolkit"
)

type Monitoring struct {
	orm.ModelBase      `bson:"-",json:"-"`
	ID                 string ` bson:"_id" , json:"_id" `
	TimeStamp          time.Time
	DateInfo           DateInfo
	LastUpdate         time.Time
	LastUpdateDateInfo DateInfo
	Project            string
	Turbine            string

	Production       float64 // MWh - Energy
	WindSpeed        float64
	PerformanceIndex float64 // skip
	MachineAvail     float64 // skip
	GridAvail        float64 // skip

	RotorSpeedRPM float64
	/*===================================*/
	IsAlarm   bool
	IsWarning bool

	Type       string // Alarm, Brake, Warning
	Status     string // ok, brake, N/A
	StatusCode int    // brake : AlarmID
	StatusDesc string // brake : AlarmDescription
}

func (m *Monitoring) New() *Monitoring {
	timeStampStr := m.TimeStamp.Format("060102_150405")
	m.ID = m.Project + "#" + m.Turbine + "#" + timeStampStr
	return m
}

func (m *Monitoring) RecordID() interface{} {
	return m.ID
}

func (m *Monitoring) TableName() string {
	return "Monitoring"
}

type MonitoringEvent struct {
	orm.ModelBase     `bson:"-",json:"-"`
	ID                string ` bson:"_id" , json:"_id" `
	Project           string
	Turbine           string
	TimeStamp         time.Time
	TimeStampStr      string
	DateInfo          DateInfo
	GroupTimeStamp    time.Time
	GroupTimeStampStr string
	AlarmId           int
	AlarmDescription  string
	Type              string // Alarm, Brake, Warning
	Status            string /// down, up
	Duration          float64
}

func (m *MonitoringEvent) New() *MonitoringEvent {
	timestampstr := m.TimeStamp.Format("060102_150405")
	// nowstr := time.Now().Format("060102_150405")
	m.ID = m.Project + "#" + m.Turbine + "#" + timestampstr + "#" + tk.ToString(m.AlarmId) + "#" + m.Status + "#" + m.Type //+ "_" + nowstr
	return m
}

func (m *MonitoringEvent) RecordID() interface{} {
	return m.ID
}

func (m *MonitoringEvent) TableName() string {
	return "MonitoringEvent"
}
<commit_msg>add some new fields for monitoring<commit_after>package models

import (
	. "eaciit/wfdemo-git/library/helper"
	"time"

	"github.com/eaciit/orm"
	tk "github.com/eaciit/toolkit"
)

type Monitoring struct {
	orm.ModelBase      `bson:"-",json:"-"`
	ID                 string ` bson:"_id" , json:"_id" `
	TimeStamp          time.Time
	DateInfo           DateInfo
	LastUpdate         time.Time
	LastUpdateDateInfo DateInfo
	Project            string
	Turbine            string

	Production       float64 // MWh - Energy
	WindSpeed        float64
	PerformanceIndex float64 // skip
	MachineAvail     float64 // skip
	GridAvail        float64 // skip

	RotorSpeedRPM float64
	/*===================================*/
	IsAlarm   bool
	IsWarning bool

	Type       string // Alarm, Brake, Warning
	Status     string // ok, brake, N/A
	StatusCode int    // brake : AlarmID
	StatusDesc string // brake : AlarmDescription
}

func (m *Monitoring) New() *Monitoring {
	timeStampStr := m.TimeStamp.Format("060102_150405")
	m.ID = m.Project + "#" + m.Turbine + "#" + timeStampStr
	return m
}

func (m *Monitoring) RecordID() interface{} {
	return m.ID
}

func (m *Monitoring) TableName() string {
	return "Monitoring"
}

type MonitoringEvent struct {
	orm.ModelBase     `bson:"-",json:"-"`
	ID                string ` bson:"_id" , json:"_id" `
	Project           string
	Turbine           string
	TimeStamp         time.Time
	TimeStampStr      string
	DateInfo          DateInfo
	GroupTimeStamp    time.Time
	GroupTimeStampStr string
	AlarmId           int
	AlarmDescription  string
	Type              string // Alarm, Brake, Warning
	Status            string /// down, up
	Duration          float64
	PitchAngle        float64
	WindDirection     float64
}

func (m *MonitoringEvent) New() *MonitoringEvent {
	timestampstr := m.TimeStamp.Format("060102_150405")
	// nowstr := time.Now().Format("060102_150405")
	m.ID = m.Project + "#" + m.Turbine + "#" + timestampstr + "#" + tk.ToString(m.AlarmId) + "#" + m.Status + "#" + m.Type //+ "_" + nowstr
	return m
}

func (m *MonitoringEvent) RecordID() interface{} {
	return m.ID
}

func (m *MonitoringEvent) TableName() string {
	return "MonitoringEvent"
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
)

type Plugin struct {
	Repo   Repo
	Build  Build
	Netrc  Netrc
	Config Config
}

func (p Plugin) Exec() error {
	if p.Build.Path != "" {
		err := os.MkdirAll(p.Build.Path, 0777)
		if err != nil {
			return err
		}
	}

	err := writeNetrc(p.Netrc.Login, p.Netrc.Login, p.Netrc.Password)
	if err != nil {
		return err
	}

	var cmds []*exec.Cmd

	if p.Config.SkipVerify {
		cmds = append(cmds, skipVerify())
	}

	if isDirEmpty(filepath.Join(p.Build.Path, ".git")) {
		cmds = append(cmds, initGit())
		cmds = append(cmds, remote(p.Repo.Clone))
	}

	switch {
	case isPullRequest(p.Build.Event) || isTag(p.Build.Event, p.Build.Ref):
		cmds = append(cmds, fetch(p.Build.Ref, p.Config.Tags, p.Config.Depth))
		cmds = append(cmds, checkoutHead())
	default:
		cmds = append(cmds, fetch(p.Build.Ref, p.Config.Tags, p.Config.Depth))
		cmds = append(cmds, checkoutSha(p.Build.Commit))
	}

	for name, url := range p.Config.Submodules {
		cmds = append(cmds, remapSubmodule(name, url))
	}

	if p.Config.Recursive {
		cmds = append(cmds, updateSubmodules(p.Config.SubmoduleRemote))
	}

	for _, cmd := range cmds {
		cmd.Dir = p.Build.Path
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		trace(cmd)
		err := cmd.Run()
		if err != nil {
			return err
		}
	}

	return nil
}

// Creates an empty git repository.
func initGit() *exec.Cmd {
	return exec.Command(
		"git",
		"init",
	)
}

// Sets the remote origin for the repository.
func remote(remote string) *exec.Cmd {
	return exec.Command(
		"git",
		"remote",
		"add",
		"origin",
		remote,
	)
}

// Checkout executes a git checkout command.
func checkoutHead() *exec.Cmd {
	return exec.Command(
		"git",
		"checkout",
		"-qf",
		"FETCH_HEAD",
	)
}

// Checkout executes a git checkout command.
func checkoutSha(commit string) *exec.Cmd {
	return exec.Command(
		"git",
		"reset",
		"--hard",
		"-q",
		commit,
	)
}

// fetch retuns git command that fetches from origin. If tags is true
// then tags will be fetched.
func fetch(ref string, tags bool, depth int) *exec.Cmd {
	tags_option := "--no-tags"
	if tags {
		tags_option = "--tags"
	}
	cmd := exec.Command(
		"git",
		"fetch",
		tags_option,
	)
	if depth != 0 {
		cmd.Args = append(cmd.Args, fmt.Sprintf("--depth=%d", depth))
	}
	cmd.Args = append(cmd.Args, "origin")
	cmd.Args = append(cmd.Args, fmt.Sprintf("+%s:", ref))
	return cmd
}

// updateSubmodules recursively initializes and updates submodules.
func updateSubmodules(remote bool) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"submodule",
		"update",
		"--init",
		"--recursive",
	)

	if remote {
		cmd.Args = append(cmd.Args, "--remote")
	}

	return cmd
}

// skipVerify returns a git command that, when executed configures git to skip
// ssl verification. This should may be used with self-signed certificates.
func skipVerify() *exec.Cmd {
	return exec.Command(
		"git",
		"config",
		"--global",
		"http.sslVerify",
		"false",
	)
}

// remapSubmodule returns a git command that, when executed configures git to
// remap submodule urls.
func remapSubmodule(name, url string) *exec.Cmd {
	name = fmt.Sprintf("submodule.%s.url", name)
	return exec.Command(
		"git",
		"config",
		name,
		url,
	)
}
<commit_msg>fixed netrc file builder<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
)

type Plugin struct {
	Repo   Repo
	Build  Build
	Netrc  Netrc
	Config Config
}

func (p Plugin) Exec() error {
	if p.Build.Path != "" {
		err := os.MkdirAll(p.Build.Path, 0777)
		if err != nil {
			return err
		}
	}

	err := writeNetrc(p.Netrc.Machine, p.Netrc.Login, p.Netrc.Password)
	if err != nil {
		return err
	}

	var cmds []*exec.Cmd

	if p.Config.SkipVerify {
		cmds = append(cmds, skipVerify())
	}

	if isDirEmpty(filepath.Join(p.Build.Path, ".git")) {
		cmds = append(cmds, initGit())
		cmds = append(cmds, remote(p.Repo.Clone))
	}

	switch {
	case isPullRequest(p.Build.Event) || isTag(p.Build.Event, p.Build.Ref):
		cmds = append(cmds, fetch(p.Build.Ref, p.Config.Tags, p.Config.Depth))
		cmds = append(cmds, checkoutHead())
	default:
		cmds = append(cmds, fetch(p.Build.Ref, p.Config.Tags, p.Config.Depth))
		cmds = append(cmds, checkoutSha(p.Build.Commit))
	}

	for name, url := range p.Config.Submodules {
		cmds = append(cmds, remapSubmodule(name, url))
	}

	if p.Config.Recursive {
		cmds = append(cmds, updateSubmodules(p.Config.SubmoduleRemote))
	}

	for _, cmd := range cmds {
		cmd.Dir = p.Build.Path
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		trace(cmd)
		err := cmd.Run()
		if err != nil {
			return err
		}
	}

	return nil
}

// Creates an empty git repository.
func initGit() *exec.Cmd {
	return exec.Command(
		"git",
		"init",
	)
}

// Sets the remote origin for the repository.
func remote(remote string) *exec.Cmd {
	return exec.Command(
		"git",
		"remote",
		"add",
		"origin",
		remote,
	)
}

// Checkout executes a git checkout command.
func checkoutHead() *exec.Cmd {
	return exec.Command(
		"git",
		"checkout",
		"-qf",
		"FETCH_HEAD",
	)
}

// Checkout executes a git checkout command.
func checkoutSha(commit string) *exec.Cmd {
	return exec.Command(
		"git",
		"reset",
		"--hard",
		"-q",
		commit,
	)
}

// fetch retuns git command that fetches from origin. If tags is true
// then tags will be fetched.
func fetch(ref string, tags bool, depth int) *exec.Cmd {
	tags_option := "--no-tags"
	if tags {
		tags_option = "--tags"
	}
	cmd := exec.Command(
		"git",
		"fetch",
		tags_option,
	)
	if depth != 0 {
		cmd.Args = append(cmd.Args, fmt.Sprintf("--depth=%d", depth))
	}
	cmd.Args = append(cmd.Args, "origin")
	cmd.Args = append(cmd.Args, fmt.Sprintf("+%s:", ref))
	return cmd
}

// updateSubmodules recursively initializes and updates submodules.
func updateSubmodules(remote bool) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"submodule",
		"update",
		"--init",
		"--recursive",
	)

	if remote {
		cmd.Args = append(cmd.Args, "--remote")
	}

	return cmd
}

// skipVerify returns a git command that, when executed configures git to skip
// ssl verification. This should may be used with self-signed certificates.
func skipVerify() *exec.Cmd {
	return exec.Command(
		"git",
		"config",
		"--global",
		"http.sslVerify",
		"false",
	)
}

// remapSubmodule returns a git command that, when executed configures git to
// remap submodule urls.
func remapSubmodule(name, url string) *exec.Cmd {
	name = fmt.Sprintf("submodule.%s.url", name)
	return exec.Command(
		"git",
		"config",
		name,
		url,
	)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/bluele/slack"
	"github.com/drone/drone-template-lib/template"
)

type (
	Repo struct {
		Owner string
		Name  string
	}

	Build struct {
		Tag      string
		Event    string
		Number   int
		Commit   string
		Ref      string
		Branch   string
		Author   Author
		Pull     string
		Message  Message
		DeployTo string
		Status   string
		Link     string
		Started  int64
		Created  int64
	}

	Author struct {
		Username string
		Name     string
		Email    string
		Avatar   string
	}

	Message struct {
		msg   string
		Title string
		Body  string
	}

	Config struct {
		Webhook          string
		Channel          string
		Recipient        string
		Username         string
		Template         string
		TemplateFallback string
		ImageURL         string
		IconURL          string
		IconEmoji        string
		LinkNames        bool
	}

	Job struct {
		Started int64
	}

	Plugin struct {
		Repo   Repo
		Build  Build
		Config Config
		Job    Job
	}
)

func (a Author) String() string {
	return a.Username
}

func newCommitMessage(m string) Message {
	// not checking the length here
	// as split will always return at least one element
	// check it if using more than the first element
	splitMsg := strings.Split(m, "\n")

	return Message{
		msg:   m,
		Title: strings.TrimSpace(splitMsg[0]),
		Body:  strings.TrimSpace(strings.Join(splitMsg[1:], "\n")),
	}
}
func (m Message) String() string {
	return m.msg
}

func (p Plugin) Exec() error {
	attachment := slack.Attachment{
		Color:      color(p.Build),
		MarkdownIn: []string{"text", "fallback"},
		ImageURL:   p.Config.ImageURL,
	}
	if p.Config.TemplateFallback != "" {
		var err error
		attachment.Fallback, err = templateMessage(p.Config.TemplateFallback, p)
		if err != nil {
			return err
		}
	} else {
		attachment.Fallback = fallback(p.Repo, p.Build)
	}

	payload := slack.WebHookPostPayload{}
	payload.Username = p.Config.Username
	payload.Attachments = []*slack.Attachment{&attachment}
	payload.IconUrl = p.Config.IconURL
	payload.IconEmoji = p.Config.IconEmoji

	if p.Config.Recipient != "" {
		payload.Channel = prepend("@", p.Config.Recipient)
	} else if p.Config.Channel != "" {
		payload.Channel = prepend("#", p.Config.Channel)
	}
	if p.Config.LinkNames == true {
		payload.LinkNames = "1"
	}
	if p.Config.Template != "" {
		var err error
		attachment.Text, err = templateMessage(p.Config.Template, p)
		if err != nil {
			return err
		}
	} else {
		attachment.Text = message(p.Repo, p.Build)
	}

	client := slack.NewWebHook(p.Config.Webhook)
	return client.PostMessage(&payload)
}

func templateMessage(t string, plugin Plugin) (string, error) {
	return template.RenderTrim(t, plugin)
}

func message(repo Repo, build Build) string {
	return fmt.Sprintf("*%s* <%s|%s/%s#%s> (%s) by %s",
		build.Status,
		build.Link,
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func fallback(repo Repo, build Build) string {
	return fmt.Sprintf("%s %s/%s#%s (%s) by %s",
		build.Status,
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func color(build Build) string {
	switch build.Status {
	case "success":
		return "good"
	case "failure", "error", "killed":
		return "danger"
	default:
		return "warning"
	}
}

func prepend(prefix, s string) string {
	if !strings.HasPrefix(s, prefix) {
		return prefix + s
	}

	return s
}
<commit_msg>Revert commit to split changes in two different prs<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/bluele/slack"
	"github.com/drone/drone-template-lib/template"
)

type (
	Repo struct {
		Owner string
		Name  string
	}

	Build struct {
		Tag      string
		Event    string
		Number   int
		Commit   string
		Ref      string
		Branch   string
		Author   Author
		Pull     string
		Message  Message
		DeployTo string
		Status   string
		Link     string
		Started  int64
		Created  int64
	}

	Author struct {
		Username string
		Name     string
		Email    string
		Avatar   string
	}

	Message struct {
		msg   string
		Title string
		Body  string
	}

	Config struct {
		Webhook          string
		Channel          string
		Recipient        string
		Username         string
		Template         string
		TemplateFallback string
		ImageURL         string
		IconURL          string
		IconEmoji        string
		Color            string
		LinkNames        bool
	}

	Job struct {
		Started int64
	}

	Plugin struct {
		Repo   Repo
		Build  Build
		Config Config
		Job    Job
	}
)

func (a Author) String() string {
	return a.Username
}

func newCommitMessage(m string) Message {
	// not checking the length here
	// as split will always return at least one element
	// check it if using more than the first element
	splitMsg := strings.Split(m, "\n")

	return Message{
		msg:   m,
		Title: strings.TrimSpace(splitMsg[0]),
		Body:  strings.TrimSpace(strings.Join(splitMsg[1:], "\n")),
	}
}
func (m Message) String() string {
	return m.msg
}

func (p Plugin) Exec() error {
	attachment := slack.Attachment{
		MarkdownIn: []string{"text", "fallback"},
		ImageURL:   p.Config.ImageURL,
	}
	if p.Config.Color == "" {
		attachment.Color = color(p.Build)
	} else {
		attachment.Color = checkAndSelectColor(p.Config.Color)
	}
	if p.Config.TemplateFallback != "" {
		var err error
		attachment.Fallback, err = templateMessage(p.Config.TemplateFallback, p)
		if err != nil {
			return err
		}
	} else {
		attachment.Fallback = fallback(p.Repo, p.Build)
	}

	payload := slack.WebHookPostPayload{}
	payload.Username = p.Config.Username
	payload.Attachments = []*slack.Attachment{&attachment}
	payload.IconUrl = p.Config.IconURL
	payload.IconEmoji = p.Config.IconEmoji

	if p.Config.Recipient != "" {
		payload.Channel = prepend("@", p.Config.Recipient)
	} else if p.Config.Channel != "" {
		payload.Channel = prepend("#", p.Config.Channel)
	}
	if p.Config.LinkNames == true {
		payload.LinkNames = "1"
	}
	if p.Config.Template != "" {
		var err error
		attachment.Text, err = templateMessage(p.Config.Template, p)
		if err != nil {
			return err
		}
	} else {
		attachment.Text = message(p.Repo, p.Build)
	}

	client := slack.NewWebHook(p.Config.Webhook)
	return client.PostMessage(&payload)
}

func templateMessage(t string, plugin Plugin) (string, error) {
	return template.RenderTrim(t, plugin)
}

func message(repo Repo, build Build) string {
	return fmt.Sprintf("*%s* <%s|%s/%s#%s> (%s) by %s",
		build.Status,
		build.Link,
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func fallback(repo Repo, build Build) string {
	return fmt.Sprintf("%s %s/%s#%s (%s) by %s",
		build.Status,
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func color(build Build) string {
	switch build.Status {
	case "success":
		return "good"
	case "failure", "error", "killed":
		return "danger"
	default:
		return "warning"
	}
}

func checkAndSelectColor(color string) string {
	for _, c := range []string{"good", "danger", "warning"} {
		if c == color {
			return color
		}
	}
	return "warning"
}

func prepend(prefix, s string) string {
	if !strings.HasPrefix(s, prefix) {
		return prefix + s
	}

	return s
}
<|endoftext|>


<commit_before>package mysqlproto

import (
	"io"
)

type Conn struct {
	*Stream
	CapabilityFlags uint32
}

func Handshake(rw io.ReadWriteCloser, capabilityFlags uint32,
	username, password, database string,
	connectAttrs map[string]string) (Conn, error) {
	stream := NewStream(rw)
	handshakeV10, err := ReadHandshakeV10(stream)
	if err != nil {
		return Conn{}, err
	}

	flags := handshakeV10.CapabilityFlags & capabilityFlags

	res := HandshakeResponse41(
		flags,
		handshakeV10.CharacterSet,
		username,
		password,
		handshakeV10.AuthPluginData,
		database,
		handshakeV10.AuthPluginName,
		connectAttrs,
	)

	conn := Conn{
		stream,
		uint32(res[4]) | uint32(res[5])<<8 | uint32(res[6])<<16 | uint32(res[7])<<24,
	}

	if _, err = conn.Write(res); err != nil {
		return conn, err
	}

	packet, err := conn.NextPacket()
	if err != nil {
		return conn, err
	}

	if packet.Payload[0] == OK_PACKET {
		return conn, nil
	} else {
		return conn, parseError(packet.Payload, conn.CapabilityFlags)
	}
}
<commit_msg>Rename Handshake to ConnectPlainHandshake<commit_after>package mysqlproto

import (
	"io"
)

type Conn struct {
	*Stream
	CapabilityFlags uint32
}

func ConnectPlainHandshake(rw io.ReadWriteCloser, capabilityFlags uint32,
	username, password, database string,
	connectAttrs map[string]string) (Conn, error) {
	stream := NewStream(rw)
	handshakeV10, err := ReadHandshakeV10(stream)
	if err != nil {
		return Conn{}, err
	}

	flags := handshakeV10.CapabilityFlags & capabilityFlags

	res := HandshakeResponse41(
		flags,
		handshakeV10.CharacterSet,
		username,
		password,
		handshakeV10.AuthPluginData,
		database,
		handshakeV10.AuthPluginName,
		connectAttrs,
	)

	conn := Conn{
		stream,
		uint32(res[4]) | uint32(res[5])<<8 | uint32(res[6])<<16 | uint32(res[7])<<24,
	}

	if _, err = conn.Write(res); err != nil {
		return conn, err
	}

	packet, err := conn.NextPacket()
	if err != nil {
		return conn, err
	}

	if packet.Payload[0] == OK_PACKET {
		return conn, nil
	} else {
		return conn, parseError(packet.Payload, conn.CapabilityFlags)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"net"
    "io"
)

type Stream struct {
    XMLName xml.Name `xml:"stream:stream"`
    Iq
}

type Query struct {
    XMLName xml.Name `xml:"query"`
    Xmlns string `xml:"xmlns,attr"`
    Username string `xml:"username"`
    Password string `xml:"password"`
    Digest string `xml:"digest"`
    Resource string `xml:"resource"`
}

type Iq struct {
    XMLName xml.Name `xml:"iq"`
	Type string `xml:"type,attr"`
	Id   string `xml:"id,attr"`
    Query
}

type LoginReq struct {
	//<?xml version='1.0' ?><stream:stream to='localhost' xmlns='jabber:client'
	//xmlns:stream='http://etherx.jabber.org/streams' version='1.0'>
	//<iq type='get' id='purple3d9c697f'><query xmlns='jabber:iq:auth'>
	//<username>james</username></query></iq></stream:stream>
	Iq Iq `xml:"iq"`
}

func Serve(addr string) {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		// handle error
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			// handle error
			continue
		}
		go handleConnection(conn)
	}
}

func handleLogin(loginRequest string) (x string, e error) {

	v := LoginReq{}

	err := xml.Unmarshal([]byte(loginRequest), &v)

	if err != nil {
		fmt.Printf("err: %v", err)
		return "", err
	}
	fmt.Printf("Id: %v\n", v.Iq.Id)
    return v.Iq.Id, err
}

func handleConnection(conn net.Conn) {
	fmt.Printf("Handling connection....")

	b := bufio.NewReader(conn)

	fmt.Fprintf(conn, `<?xml version='1.0'?>
		<stream:stream
			from='localhost'
			id='abc123'
			to='james@localhost'
			version='1.0'
			xml:lang='en'
			xmlns='jabber:server'
			xmlns:stream='http://etherx.jabber.org/streams'>`)

	fmt.Fprintf(conn, "<stream:features/>")
	for {
		line, _, err := b.ReadLine()
		if err != nil {
			break
		}
		loginId, err := handleLogin(string(line))
        fmt.Println(loginId)
		fmt.Println(string(line))
        r := &Stream{}
        r.Iq = Iq{Id: loginId, Type: "result"}
        r.Iq.Query = Query{Xmlns: "jabber:iq:auth"}
        output, err := xml.Marshal(r)
        fmt.Println(string(output))

        line, _, err = b.ReadLine()
        loginId2, err := handleLogin(string(line))
        i := &Stream{}
        i.Iq = Iq{Id: loginId2, Type: "result"}
        output2, err := xml.Marshal(i)
        fmt.Println(string(output2))
        //fmt.Fprintf(conn, string(output))
	}

}
<commit_msg>Basic login hacking<commit_after>package main

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"net"
_    "io"
)

type Stream struct {
    XMLName xml.Name `xml:"stream:stream"`
    Iq
}

type Query struct {
    XMLName xml.Name `xml:"query"`
    Xmlns string `xml:"xmlns,attr"`
    Username string `xml:"username"`
    Password string `xml:"password"`
    Digest string `xml:"digest"`
    Resource string `xml:"resource"`
}

type Iq struct {
    XMLName xml.Name `xml:"iq"`
	Type string `xml:"type,attr"`
	Id   string `xml:"id,attr"`
    Query
}

type LoginReq struct {
	//<?xml version='1.0' ?><stream:stream to='localhost' xmlns='jabber:client'
	//xmlns:stream='http://etherx.jabber.org/streams' version='1.0'>
	//<iq type='get' id='purple3d9c697f'><query xmlns='jabber:iq:auth'>
	//<username>james</username></query></iq></stream:stream>
	Iq Iq `xml:"iq"`
}

func Serve(addr string) {
    fmt.Println("Server started")
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		// handle error
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			// handle error
			continue
		}
		go handleConnection(conn)
	}
}

func handleLogin(loginRequest string) (x string, e error) {

	v := LoginReq{}

	err := xml.Unmarshal([]byte(loginRequest), &v)

	if err != nil {
		fmt.Printf("err: %v", err)
		return "", err
	}
	fmt.Printf("Id: %v\n", v.Iq.Id)
    return v.Iq.Id, err
}

func handleConnection(conn net.Conn) {
	fmt.Printf("Handling connection....")

	b := bufio.NewReader(conn)

	fmt.Fprintf(conn, `<?xml version='1.0'?>
		<stream:stream
			from='localhost'
			id='abc123'
			to='james@localhost'
			version='1.0'
			xml:lang='en'
			xmlns='jabber:server'
			xmlns:stream='http://etherx.jabber.org/streams'>`)

	fmt.Fprintf(conn, "<stream:features/>")
	for {
		line, _, err := b.ReadLine()
		if err != nil {
			break
		}
		loginId, err := handleLogin(string(line))
        fmt.Println(loginId)
		fmt.Println(string(line))
        r := &Stream{}
        r.Iq = Iq{Id: loginId, Type: "result"}
        r.Iq.Query = Query{Xmlns: "jabber:iq:auth"}
        output, err := xml.Marshal(r)
        fmt.Println(string(output))

        line, _, err = b.ReadLine()
        loginId2, err := handleLogin(string(line))
        i := &Stream{}
        i.Iq = Iq{Id: loginId2, Type: "result"}
        output2, err := xml.Marshal(i)
        fmt.Println(string(output2))
        //fmt.Fprintf(conn, string(output))
	}

}
<|endoftext|>