<commit_before>// Package app provides application support for context and MongoDB access.
// Current Status Codes:
//		200 OK           : StatusOK                  : Call is success and returning data.
//		204 No Content   : StatusNoContent           : Call is success and returns no data.
//		400 Bad Request  : StatusBadRequest          : Invalid post data (syntax or semantics).
//		401 Unauthorized : StatusUnauthorized        : Authentication failure.
//		404 Not Found    : StatusNotFound            : Invalid URL or identifier.
//		500 Internal     : StatusInternalServerError : Application specific beyond scope of user.
package app

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/ardanlabs/kit/log"
)

// Invalid describes a validation error belonging to a specific field.
type Invalid struct {
	Fld string `json:"field_name"`
	Err string `json:"error"`
}

// jsonError is the response for errors that occur within the API.
type jsonError struct {
	Error  string    `json:"error"`
	Fields []Invalid `json:"fields,omitempty"`
}

//==============================================================================

// Context contains data associated with a single request.
type Context struct {
	http.ResponseWriter
	Request   *http.Request
	Now       time.Time
	Params    map[string]string
	SessionID string
	Status    int
	Ctx       map[string]interface{}
	App       *App
}

// Error handles all error responses for the API.
func (c *Context) Error(err error) {
	switch err {
	case ErrNotFound:
		c.RespondError(err.Error(), http.StatusNotFound)
	case ErrInvalidID:
		c.RespondError(err.Error(), http.StatusBadRequest)
	case ErrValidation:
		c.RespondError(err.Error(), http.StatusBadRequest)
	case ErrNotAuthorized:
		c.RespondError(err.Error(), http.StatusUnauthorized)
	default:
		c.RespondError(err.Error(), http.StatusInternalServerError)
	}
}

// Respond sends JSON to the client.
// If code is StatusNoContent, v is expected to be nil.
func (c *Context) Respond(data interface{}, code int) {
	log.User(c.SessionID, "api : Respond", "Started : Code[%d]", code)

	c.Status = code

	// Load any user defined header values.
	if app.userHeaders != nil {
		for key, value := range app.userHeaders {
			log.User(c.SessionID, "api : Respond", "Setting user headers : %s:%s", key, value)
			c.Header().Set(key, value)
		}
	}

	// Just set the status code and we are done.
	if code == http.StatusNoContent {
		c.WriteHeader(code)
		return
	}

	// Set the content type.
	c.Header().Set("Content-Type", "application/json")

	// Write the status code.
	c.WriteHeader(code)

	// Marshal the data into a JSON string.
	jsonData, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		log.Error(c.SessionID, "api : Respond", err, "Marshalling JSON response")
		jsonData = []byte("{}")
	}

	// Look for a JSONP marker
	if cb := c.Request.URL.Query().Get("callback"); cb != "" {

		// We need to wrap the result in a function call.
		// callback_value({"data_1": "hello world", "data_2": ["the","sun","is","shining"]});
		io.WriteString(c, cb+"("+string(jsonData)+")")

		log.User(c.SessionID, "api : Respond", "Completed")
		return
	}

	// We can send the result straight through.
	io.WriteString(c, string(jsonData))

	log.User(c.SessionID, "api : Respond", "Completed")
}

// RespondInvalid sends JSON describing field validation errors.
func (c *Context) RespondInvalid(fields []Invalid) {
	v := jsonError{
		Error:  "field validation failure",
		Fields: fields,
	}
	c.Respond(v, http.StatusBadRequest)
}

// RespondError sends JSON describing the error
func (c *Context) RespondError(error string, code int) {
	c.Respond(jsonError{Error: error}, code)
}

// Proxy will setup a direct proxy inbetween this service and the destination
// service. If the targetPath is empty, the path on the target will be set to
// the targetURL's path concatenated with the request path.
func (c *Context) Proxy(targetURL, targetPath string) error {
	target, err := url.Parse(targetURL)
	if err != nil {
		return err
	}

	// If the targetPath was not specified, then we need to source it from the
	// request path and the target path together.
	if targetPath == "" {
		targetPath = singleJoiningSlash(target.Path, c.Request.URL.Path)
	}

	// Define our custom request director to ensure that the correct headers are
	// forwarded as well as having the request path and query rewritten
	// properly.
	director := func(req *http.Request) {
		req.URL.Scheme = target.Scheme
		req.URL.Host = target.Host
		req.URL.Path = targetPath

		if target.RawQuery == "" || req.URL.RawQuery == "" {
			req.URL.RawQuery = target.RawQuery + req.URL.RawQuery
		} else {
			req.URL.RawQuery = target.RawQuery + "&" + req.URL.RawQuery
		}

		// Set the headers on the incoming request by clearing them all out.
		req.Header = make(http.Header)

		// Set each of the expected headers that we want.
		req.Header.Set("Content-Type", "application/json")

		// TODO: add authorization headers here.
	}

	// Create a new reverse proxy. We need to to this here because for the path
	// rewriting we may need access to variables stored in this specific
	// request's path parameters which can allow to be overridden via the
	// rewrite argument to this function.
	proxy := httputil.ReverseProxy{Director: director}

	// Create a new proxy response writer that will record the http status code
	// issued by the reverse proxy.
	prw := ProxyResponseWriter{
		ResponseWriter: c.ResponseWriter,
	}

	// Serve the request via the built in handler here.
	proxy.ServeHTTP(&prw, c.Request)

	c.Status = prw.Status

	return nil
}

// singleJoiningSlash ensures that there is a single joining slash inbetween the
// url's that are being joined. This was sourced from the reverseproxy.go file
// inside the net/http/httputil package in the stdlib.
func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}
<commit_msg>Change to Proxy to allow for custom rewriting logic<commit_after>// Package app provides application support for context and MongoDB access.
// Current Status Codes:
//		200 OK           : StatusOK                  : Call is success and returning data.
//		204 No Content   : StatusNoContent           : Call is success and returns no data.
//		400 Bad Request  : StatusBadRequest          : Invalid post data (syntax or semantics).
//		401 Unauthorized : StatusUnauthorized        : Authentication failure.
//		404 Not Found    : StatusNotFound            : Invalid URL or identifier.
//		500 Internal     : StatusInternalServerError : Application specific beyond scope of user.
package app

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/ardanlabs/kit/log"
)

// Invalid describes a validation error belonging to a specific field.
type Invalid struct {
	Fld string `json:"field_name"`
	Err string `json:"error"`
}

// jsonError is the response for errors that occur within the API.
type jsonError struct {
	Error  string    `json:"error"`
	Fields []Invalid `json:"fields,omitempty"`
}

//==============================================================================

// Context contains data associated with a single request.
type Context struct {
	http.ResponseWriter
	Request   *http.Request
	Now       time.Time
	Params    map[string]string
	SessionID string
	Status    int
	Ctx       map[string]interface{}
	App       *App
}

// Error handles all error responses for the API.
func (c *Context) Error(err error) {
	switch err {
	case ErrNotFound:
		c.RespondError(err.Error(), http.StatusNotFound)
	case ErrInvalidID:
		c.RespondError(err.Error(), http.StatusBadRequest)
	case ErrValidation:
		c.RespondError(err.Error(), http.StatusBadRequest)
	case ErrNotAuthorized:
		c.RespondError(err.Error(), http.StatusUnauthorized)
	default:
		c.RespondError(err.Error(), http.StatusInternalServerError)
	}
}

// Respond sends JSON to the client.
// If code is StatusNoContent, v is expected to be nil.
func (c *Context) Respond(data interface{}, code int) {
	log.User(c.SessionID, "api : Respond", "Started : Code[%d]", code)

	c.Status = code

	// Load any user defined header values.
	if app.userHeaders != nil {
		for key, value := range app.userHeaders {
			log.User(c.SessionID, "api : Respond", "Setting user headers : %s:%s", key, value)
			c.Header().Set(key, value)
		}
	}

	// Just set the status code and we are done.
	if code == http.StatusNoContent {
		c.WriteHeader(code)
		return
	}

	// Set the content type.
	c.Header().Set("Content-Type", "application/json")

	// Write the status code.
	c.WriteHeader(code)

	// Marshal the data into a JSON string.
	jsonData, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		log.Error(c.SessionID, "api : Respond", err, "Marshalling JSON response")
		jsonData = []byte("{}")
	}

	// Look for a JSONP marker
	if cb := c.Request.URL.Query().Get("callback"); cb != "" {

		// We need to wrap the result in a function call.
		// callback_value({"data_1": "hello world", "data_2": ["the","sun","is","shining"]});
		io.WriteString(c, cb+"("+string(jsonData)+")")

		log.User(c.SessionID, "api : Respond", "Completed")
		return
	}

	// We can send the result straight through.
	io.WriteString(c, string(jsonData))

	log.User(c.SessionID, "api : Respond", "Completed")
}

// RespondInvalid sends JSON describing field validation errors.
func (c *Context) RespondInvalid(fields []Invalid) {
	v := jsonError{
		Error:  "field validation failure",
		Fields: fields,
	}
	c.Respond(v, http.StatusBadRequest)
}

// RespondError sends JSON describing the error
func (c *Context) RespondError(error string, code int) {
	c.Respond(jsonError{Error: error}, code)
}

// Proxy will setup a direct proxy inbetween this service and the destination
// service.
func (c *Context) Proxy(targetURL string, rewrite func(req *http.Request)) error {
	target, err := url.Parse(targetURL)
	if err != nil {
		return err
	}

	// Define our custom request director to ensure that the correct headers are
	// forwarded as well as having the request path and query rewritten
	// properly.
	director := func(req *http.Request) {
		req.URL.Scheme = target.Scheme
		req.URL.Host = target.Host
		req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)

		if target.RawQuery == "" || req.URL.RawQuery == "" {
			req.URL.RawQuery = target.RawQuery + req.URL.RawQuery
		} else {
			req.URL.RawQuery = target.RawQuery + "&" + req.URL.RawQuery
		}

		// Set the headers on the incoming request by clearing them all out.
		req.Header = make(http.Header)

		// Rewrite the request for the director if a rewrite function was passed.
		if rewrite != nil {
			rewrite(req)
		}
	}

	// Create a new reverse proxy. We need to to this here because for the path
	// rewriting we may need access to variables stored in this specific
	// request's path parameters which can allow to be overridden via the
	// rewrite argument to this function.
	proxy := httputil.ReverseProxy{Director: director}

	// Create a new proxy response writer that will record the http status code
	// issued by the reverse proxy.
	prw := ProxyResponseWriter{
		ResponseWriter: c.ResponseWriter,
	}

	// Serve the request via the built in handler here.
	proxy.ServeHTTP(&prw, c.Request)

	c.Status = prw.Status

	return nil
}

// singleJoiningSlash ensures that there is a single joining slash inbetween the
// url's that are being joined. This was sourced from the reverseproxy.go file
// inside the net/http/httputil package in the stdlib.
func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}
<|endoftext|>


<commit_before>package mesos

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"sync"

	"github.com/CiscoCloud/mesos-consul/config"
	"github.com/CiscoCloud/mesos-consul/consul"
	"github.com/CiscoCloud/mesos-consul/registry"
	"github.com/CiscoCloud/mesos-consul/state"

	consulapi "github.com/hashicorp/consul/api"
	proto "github.com/mesos/mesos-go/mesosproto"
	log "github.com/sirupsen/logrus"
)

type CacheEntry struct {
	service      *consulapi.AgentServiceRegistration
	isRegistered bool
}

type Mesos struct {
	Registry registry.Registry
	Agents   map[string]string
	Lock     sync.Mutex

	Leader    *proto.MasterInfo
	Masters   []*proto.MasterInfo
	started   sync.Once
	startChan chan struct{}

	IpOrder        []string
	WhiteList      string
	whitelistRegex *regexp.Regexp
	BlackList      string
	blacklistRegex *regexp.Regexp

	Separator string

	ServiceName string
	ServiceTags []string
}

func New(c *config.Config) *Mesos {
	m := new(Mesos)

	if c.Zk == "" {
		return nil
	}
	m.Separator = c.Separator

	if len(c.WhiteList) > 0 {
		m.WhiteList = strings.Join(c.WhiteList, "|")
		log.WithField("whitelist", m.WhiteList).Debug("Using whitelist regex")
		re, err := regexp.Compile(m.WhiteList)
		if err != nil {
			// For now, exit if the regex fails to compile. If we read regexes from Consul
			// maybe we emit a warning and use the old regex
			//
			log.WithField("whitelist", m.WhiteList).Fatal("WhiteList regex failed to compile")
		}
		m.whitelistRegex = re
	} else {
		m.whitelistRegex = nil
	}

	if len(c.BlackList) > 0 {
		m.BlackList = strings.Join(c.BlackList, "|")
		log.WithField("blacklist", m.BlackList).Debug("Using blacklist regex")
		re, err := regexp.Compile(m.BlackList)
		if err != nil {
			// For now, exit if the regex fails to compile. If we read regexes from Consul
			// maybe we emit a warning and use the old regex
			//
			log.WithField("blacklist", m.BlackList).Fatal("BlackList regex failed to compile")
		}
		m.blacklistRegex = re
	} else {
		m.blacklistRegex = nil
	}

	m.ServiceName = cleanName(c.ServiceName, c.Separator)

	m.Registry = consul.New()

	if m.Registry == nil {
		log.Fatal("No registry specified")
	}

	m.zkDetector(c.Zk)

	m.IpOrder = strings.Split(c.MesosIpOrder, ",")
	for _, src := range m.IpOrder {
		switch src {
		case "netinfo", "host", "docker", "mesos":
		default:
			log.Fatalf("Invalid IP Search Order: '%v'", src)
		}
	}
	log.Debugf("m.IpOrder = '%v'", m.IpOrder)

	if c.ServiceTags != "" {
		m.ServiceTags = strings.Split(c.ServiceTags, ",")
	}

	return m
}

func (m *Mesos) Refresh() error {
	sj, err := m.loadState()
	if err != nil {
		log.Warn("loadState failed: ", err.Error())
		return err
	}

	if sj.Leader == "" {
		return errors.New("Empty master")
	}

	if m.Registry.CacheCreate() {
		m.LoadCache()
	}

	m.parseState(sj)

	return nil
}

func (m *Mesos) loadState() (state.State, error) {
	var err error
	var sj state.State

	log.Debug("loadState() called")

	defer func() {
		if rec := recover(); rec != nil {
			err = errors.New("can't connect to Mesos")
		}
	}()

	mh := m.getLeader()
	if mh.Ip == "" {
		log.Warn("No master in zookeeper")
		return sj, errors.New("No master in zookeeper")
	}

	log.Infof("Zookeeper leader: %s:%s", mh.Ip, mh.PortString)

	log.Info("reloading from master ", mh.Ip)
	sj = m.loadFromMaster(mh.Ip, mh.PortString)

	if rip := leaderIP(sj.Leader); rip != mh.Ip {
		log.Warn("master changed to ", rip)
		sj = m.loadFromMaster(rip, mh.PortString)
	}

	return sj, err
}

func (m *Mesos) loadFromMaster(ip string, port string) (sj state.State) {
	url := "http://" + ip + ":" + port + "/master/state.json"

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(body, &sj)
	if err != nil {
		log.Fatal(err)
	}

	return sj
}

func (m *Mesos) parseState(sj state.State) {
	log.Info("Running parseState")

	m.RegisterHosts(sj)
	log.Debug("Done running RegisterHosts")

	for _, fw := range sj.Frameworks {
		for _, task := range fw.Tasks {
			agent, ok := m.Agents[task.SlaveID]
			if ok && task.State == "TASK_RUNNING" {
				task.SlaveIP = agent
				m.registerTask(&task, agent)
			}
		}
	}

	m.Registry.Deregister()
}
<commit_msg>Preventing exit when mesos is unavailable<commit_after>package mesos

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"sync"

	"github.com/CiscoCloud/mesos-consul/config"
	"github.com/CiscoCloud/mesos-consul/consul"
	"github.com/CiscoCloud/mesos-consul/registry"
	"github.com/CiscoCloud/mesos-consul/state"

	consulapi "github.com/hashicorp/consul/api"
	proto "github.com/mesos/mesos-go/mesosproto"
	log "github.com/sirupsen/logrus"
)

type CacheEntry struct {
	service      *consulapi.AgentServiceRegistration
	isRegistered bool
}

type Mesos struct {
	Registry registry.Registry
	Agents   map[string]string
	Lock     sync.Mutex

	Leader    *proto.MasterInfo
	Masters   []*proto.MasterInfo
	started   sync.Once
	startChan chan struct{}

	IpOrder        []string
	WhiteList      string
	whitelistRegex *regexp.Regexp
	BlackList      string
	blacklistRegex *regexp.Regexp

	Separator string

	ServiceName string
	ServiceTags []string
}

func New(c *config.Config) *Mesos {
	m := new(Mesos)

	if c.Zk == "" {
		return nil
	}
	m.Separator = c.Separator

	if len(c.WhiteList) > 0 {
		m.WhiteList = strings.Join(c.WhiteList, "|")
		log.WithField("whitelist", m.WhiteList).Debug("Using whitelist regex")
		re, err := regexp.Compile(m.WhiteList)
		if err != nil {
			// For now, exit if the regex fails to compile. If we read regexes from Consul
			// maybe we emit a warning and use the old regex
			//
			log.WithField("whitelist", m.WhiteList).Fatal("WhiteList regex failed to compile")
		}
		m.whitelistRegex = re
	} else {
		m.whitelistRegex = nil
	}

	if len(c.BlackList) > 0 {
		m.BlackList = strings.Join(c.BlackList, "|")
		log.WithField("blacklist", m.BlackList).Debug("Using blacklist regex")
		re, err := regexp.Compile(m.BlackList)
		if err != nil {
			// For now, exit if the regex fails to compile. If we read regexes from Consul
			// maybe we emit a warning and use the old regex
			//
			log.WithField("blacklist", m.BlackList).Fatal("BlackList regex failed to compile")
		}
		m.blacklistRegex = re
	} else {
		m.blacklistRegex = nil
	}

	m.ServiceName = cleanName(c.ServiceName, c.Separator)

	m.Registry = consul.New()

	if m.Registry == nil {
		log.Fatal("No registry specified")
	}

	m.zkDetector(c.Zk)

	m.IpOrder = strings.Split(c.MesosIpOrder, ",")
	for _, src := range m.IpOrder {
		switch src {
		case "netinfo", "host", "docker", "mesos":
		default:
			log.Fatalf("Invalid IP Search Order: '%v'", src)
		}
	}
	log.Debugf("m.IpOrder = '%v'", m.IpOrder)

	if c.ServiceTags != "" {
		m.ServiceTags = strings.Split(c.ServiceTags, ",")
	}

	return m
}

func (m *Mesos) Refresh() error {
	sj, err := m.loadState()
	if err != nil {
		log.Warn("loadState failed: ", err.Error())
		return err
	}

	if sj.Leader == "" {
		return errors.New("Empty master")
	}

	if m.Registry.CacheCreate() {
		m.LoadCache()
	}

	m.parseState(sj)

	return nil
}

func (m *Mesos) loadState() (state.State, error) {
	var err error
	var sj state.State

	log.Debug("loadState() called")

	defer func() {
		if rec := recover(); rec != nil {
			err = errors.New("can't connect to Mesos")
		}
	}()

	mh := m.getLeader()
	if mh.Ip == "" {
		log.Warn("No master in zookeeper")
		return sj, errors.New("No master in zookeeper")
	}

	log.Infof("Zookeeper leader: %s:%s", mh.Ip, mh.PortString)

	log.Info("reloading from master ", mh.Ip)
	sj, err = m.loadFromMaster(mh.Ip, mh.PortString)

	if rip := leaderIP(sj.Leader); rip != mh.Ip {
		log.Warn("master changed to ", rip)
		sj, err = m.loadFromMaster(rip, mh.PortString)
	}

	return sj, err
}

func (m *Mesos) loadFromMaster(ip string, port string) (sj state.State, err error) {
	url := "http://" + ip + ":" + port + "/master/state.json"

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	err = json.Unmarshal(body, &sj)
	if err != nil {
		return
	}

	return sj, nil
}

func (m *Mesos) parseState(sj state.State) {
	log.Info("Running parseState")

	m.RegisterHosts(sj)
	log.Debug("Done running RegisterHosts")

	for _, fw := range sj.Frameworks {
		for _, task := range fw.Tasks {
			agent, ok := m.Agents[task.SlaveID]
			if ok && task.State == "TASK_RUNNING" {
				task.SlaveIP = agent
				m.registerTask(&task, agent)
			}
		}
	}

	m.Registry.Deregister()
}
<|endoftext|>


<commit_before>package raft

type Entry struct {
	Term int
	Data []byte
}

type log struct {
	ents    []Entry
	commit  int
	applied int
}

func newLog() *log {
	return &log{
		ents:    make([]Entry, 1, 1024),
		commit:  0,
		applied: 0,
	}
}

func (l *log) maybeAppend(index, logTerm, commit int, ents ...Entry) bool {
	if l.matchTerm(index, logTerm) {
		l.append(index, ents...)
		l.commit = commit
		return true
	}
	return false
}

func (l *log) append(after int, ents ...Entry) int {
	l.ents = append(l.ents[:after+1], ents...)
	return len(l.ents) - 1
}

func (l *log) lastIndex() int {
	return len(l.ents) - 1
}

func (l *log) term(i int) int {
	if i > l.lastIndex() {
		return -1
	}
	return l.ents[i].Term
}

func (l *log) entries(i int) []Entry {
	if i > l.lastIndex() {
		return nil
	}
	return l.ents[i:]
}

func (l *log) isUpToDate(i, term int) bool {
	e := l.ents[l.lastIndex()]
	return term > e.Term || (term == e.Term && i >= l.lastIndex())
}

func (l *log) matchTerm(i, term int) bool {
	if i > l.lastIndex() {
		return false
	}
	return l.ents[i].Term == term
}

func (l *log) maybeCommit(maxIndex, term int) bool {
	if maxIndex > l.commit && l.term(maxIndex) == term {
		l.commit = maxIndex
		return true
	}
	return false
}

func (l *log) nextEnts() (ents []Entry) {
	if l.commit > l.applied {
		ents = l.ents[l.applied+1 : l.commit+1]
		l.applied = l.commit
	}
	return ents
}
<commit_msg>raft: remove the init cap of log entries<commit_after>package raft

type Entry struct {
	Term int
	Data []byte
}

type log struct {
	ents    []Entry
	commit  int
	applied int
}

func newLog() *log {
	return &log{
		ents:    make([]Entry, 1),
		commit:  0,
		applied: 0,
	}
}

func (l *log) maybeAppend(index, logTerm, commit int, ents ...Entry) bool {
	if l.matchTerm(index, logTerm) {
		l.append(index, ents...)
		l.commit = commit
		return true
	}
	return false
}

func (l *log) append(after int, ents ...Entry) int {
	l.ents = append(l.ents[:after+1], ents...)
	return len(l.ents) - 1
}

func (l *log) lastIndex() int {
	return len(l.ents) - 1
}

func (l *log) term(i int) int {
	if i > l.lastIndex() {
		return -1
	}
	return l.ents[i].Term
}

func (l *log) entries(i int) []Entry {
	if i > l.lastIndex() {
		return nil
	}
	return l.ents[i:]
}

func (l *log) isUpToDate(i, term int) bool {
	e := l.ents[l.lastIndex()]
	return term > e.Term || (term == e.Term && i >= l.lastIndex())
}

func (l *log) matchTerm(i, term int) bool {
	if i > l.lastIndex() {
		return false
	}
	return l.ents[i].Term == term
}

func (l *log) maybeCommit(maxIndex, term int) bool {
	if maxIndex > l.commit && l.term(maxIndex) == term {
		l.commit = maxIndex
		return true
	}
	return false
}

func (l *log) nextEnts() (ents []Entry) {
	if l.commit > l.applied {
		ents = l.ents[l.applied+1 : l.commit+1]
		l.applied = l.commit
	}
	return ents
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"path/filepath"
	"strings"
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	pkgutil "k8s.io/minikube/pkg/util"

	"k8s.io/minikube/test/integration/util"
)

func testClusterDNS(t *testing.T) {
	t.Parallel()

	kubectlRunner := util.NewKubectlRunner(t)
	podPath := filepath.Join(*testdataDir, "busybox.yaml")

	client, err := pkgutil.GetClient()
	if err != nil {
		t.Fatalf("Error getting kubernetes client %s", err)
	}

	if _, err := kubectlRunner.RunCommand([]string{"create", "-f", podPath}); err != nil {
		t.Fatalf("creating busybox pod: %s", err)
	}

	if err := util.WaitForBusyboxRunning(t, "default"); err != nil {
		t.Fatalf("Waiting for busybox pod to be up: %s", err)
	}
	listOpts := metav1.ListOptions{LabelSelector: "integration-test=busybox"}
	pods, err := client.CoreV1().Pods("default").List(listOpts)
	if len(pods.Items) == 0 {
		t.Fatal("Expected a busybox pod to be running")
	}

	podName := pods.Items[0].Name
	defer kubectlRunner.RunCommand([]string{"delete", "po", podName})

	dnsByteArr, err := kubectlRunner.RunCommand([]string{"exec", podName,
		"nslookup", "kubernetes"})
	if err != nil {
		t.Fatalf("running nslookup in pod:%s", err)
	}
	dnsOutput := string(dnsByteArr)
	if !strings.Contains(dnsOutput, "10.96.0.1") || !strings.Contains(dnsOutput, "10.96.0.10") {
		t.Errorf("DNS lookup failed, could not find both 10.96.0.1 and 10.96.0.10.  Output: %s", dnsOutput)
	}
}
<commit_msg>Wait for k8s-app=kube-dns pods to be Running before testing.<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bytes"
	"path/filepath"
	"testing"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"

	commonutil "k8s.io/minikube/pkg/util"
	pkgutil "k8s.io/minikube/pkg/util"
	"k8s.io/minikube/test/integration/util"
)

func testClusterDNS(t *testing.T) {
	t.Parallel()
	client, err := pkgutil.GetClient()
	if err != nil {
		t.Fatalf("Error getting kubernetes client %s", err)
	}
	waitForDNS(t, client)

	kr := util.NewKubectlRunner(t)
	busybox := busyBoxPod(t, client, kr)
	defer kr.RunCommand([]string{"delete", "po", busybox})

	// The query result is not as important as service reachability
	out, err := kr.RunCommand([]string{"exec", busybox, "nslookup", "localhost"})
	if err != nil {
		t.Errorf("nslookup within busybox failed: %s", err)
	}
	clusterIP := []byte("10.96.0.1")
	if !bytes.Contains(out, clusterIP) {
		t.Errorf("nslookup did not mention %s:\n%s", clusterIP, out)
	}
}

func waitForDNS(t *testing.T, c kubernetes.Interface) {
	// Implementation note: both kube-dns and coredns have k8s-app=kube-dns labels.
	sel := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
	if err := commonutil.WaitForPodsWithLabelRunning(c, "kube-system", sel); err != nil {
		t.Fatalf("Waited too long for k8s-app=kube-dns pods")
	}
	if err := commonutil.WaitForDeploymentToStabilize(c, "kube-system", "kube-dns", time.Minute*2); err != nil {
		t.Fatalf("kube-dns deployment failed became stable within 2 minutes")
	}
}

func busyBoxPod(t *testing.T, c kubernetes.Interface, kr *util.KubectlRunner) string {
	if _, err := kr.RunCommand([]string{"create", "-f", filepath.Join(*testdataDir, "busybox.yaml")}); err != nil {
		t.Fatalf("creating busybox pod: %s", err)
	}
	// TODO(tstromberg): Refactor WaitForBusyboxRunning to return name of pod.
	if err := util.WaitForBusyboxRunning(t, "default"); err != nil {
		t.Fatalf("Waiting for busybox pod to be up: %s", err)
	}

	pods, err := c.CoreV1().Pods("default").List(metav1.ListOptions{LabelSelector: "integration-test=busybox"})
	if err != nil {
		t.Fatalf("list error: %v", err)
	}
	if len(pods.Items) == 0 {
		t.Fatal("Expected a busybox pod to be running")
	}
	return pods.Items[0].Name
}
<|endoftext|>


<commit_before>package builder

// nginx
const (
	NginxVersion           = "1.17.3"
	NginxDownloadURLPrefix = "https://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "8.42"
	PcreDownloadURLPrefix = "https://ftp.pcre.org/pub/pcre"
)

// openssl
const (
	OpenSSLVersion           = "1.0.2r"
	OpenSSLDownloadURLPrefix = "https://www.openssl.org/source"
)

// libressl
const (
	LibreSSLVersion           = "2.8.3"
	LibreSSLDownloadURLPrefix = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL"
)

// zlib
const (
	ZlibVersion           = "1.2.11"
	ZlibDownloadURLPrefix = "https://zlib.net/fossils"
)

// openResty
const (
	OpenRestyVersion           = "1.15.8.1"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.3.0"
	TengineDownloadURLPrefix = "https://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentLibreSSL
	ComponentZlib
	ComponentMax
)
<commit_msg>bumped default pcre version to 8.43.<commit_after>package builder

// nginx
const (
	NginxVersion           = "1.17.3"
	NginxDownloadURLPrefix = "https://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "8.43"
	PcreDownloadURLPrefix = "https://ftp.pcre.org/pub/pcre"
)

// openssl
const (
	OpenSSLVersion           = "1.0.2r"
	OpenSSLDownloadURLPrefix = "https://www.openssl.org/source"
)

// libressl
const (
	LibreSSLVersion           = "2.8.3"
	LibreSSLDownloadURLPrefix = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL"
)

// zlib
const (
	ZlibVersion           = "1.2.11"
	ZlibDownloadURLPrefix = "https://zlib.net/fossils"
)

// openResty
const (
	OpenRestyVersion           = "1.15.8.1"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.3.0"
	TengineDownloadURLPrefix = "https://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentLibreSSL
	ComponentZlib
	ComponentMax
)
<|endoftext|>


<commit_before>package configuration

import (
	"flag"
	"log"

	"github.com/FogCreek/mini"
)

type iagoConfiguration struct {
	Hostname string
	Protocol string
	Port     string
	Path     string
}

var (
	Iago iagoConfiguration
)

func Process() {
	path := flag.String("config", "/etc/miloud.ini", "Configuration file path")
	flag.Parse()

	config, err := mini.LoadConfiguration(*path)

	if err != nil {
		log.Fatal(err)
	}

	Iago.Hostname = config.StringFromSection("Iago", "Hostname", "localhost")
	Iago.Protocol = config.StringFromSection("Iago", "Protocol", "http")
	Iago.Path = config.StringFromSection("Iago", "Path", "/")
	Iago.Port = config.IntegerFromSection("Iago", "Port", 0)

	if Iago.Port == 0 {
		if Iago.Protocol == "http" {
			Iago.Port = 80
		} else if Iago.Protocol == "https" {
			Iago.Port = 443
		}
	}
}
<commit_msg>Changed the Port field type to int<commit_after>package configuration

import (
	"flag"
	"log"

	"github.com/FogCreek/mini"
)

type iagoConfiguration struct {
	Hostname string
	Protocol string
	Port     int
	Path     string
}

var (
	Iago iagoConfiguration
)

func Process() {
	path := flag.String("config", "/etc/miloud.ini", "Configuration file path")
	flag.Parse()

	config, err := mini.LoadConfiguration(*path)

	if err != nil {
		log.Fatal(err)
	}

	Iago.Hostname = config.StringFromSection("Iago", "Hostname", "localhost")
	Iago.Protocol = config.StringFromSection("Iago", "Protocol", "http")
	Iago.Path = config.StringFromSection("Iago", "Path", "/")
	Iago.Port = config.IntegerFromSection("Iago", "Port", 0)

	if Iago.Port == 0 {
		if Iago.Protocol == "http" {
			Iago.Port = 80
		} else if Iago.Protocol == "https" {
			Iago.Port = 443
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"../vendor/gocui"
)

func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if v, err := g.SetView("side", -1, -1, 5, maxY); err != nil {
	    if err != gocui.ErrorUnkView {
	        return err
	    }

	    fmt.Fprintln(v, "- 1")
	}

	if v, err := g.SetView("intro", 5, -1, maxX, maxY); err != nil {
        if err != gocui.ErrorUnkView {
            return err
        }
        fmt.Fprintf(v, "%s",`

			        GGGGGGGGGGGGG                 EEEEEEEEEEEEEEEEEEEEEE
			     GGG::::::::::::G                 E::::::::::::::::::::E
			   GG:::::::::::::::G                 E::::::::::::::::::::E
			  G:::::GGGGGGGG::::G                 EE::::::EEEEEEEEE::::E
			 G:::::G       GGGGGG   ooooooooooo     E:::::E       EEEEEE
			G:::::G               oo:::::::::::oo   E:::::E
			G:::::G              o:::::::::::::::o  E::::::EEEEEEEEEE
			G:::::G    GGGGGGGGGGo:::::ooooo:::::o  E:::::::::::::::E
			G:::::G    G::::::::Go::::o     o::::o  E:::::::::::::::E
			G:::::G    GGGGG::::Go::::o     o::::o  E::::::EEEEEEEEEE
			G:::::G        G::::Go::::o     o::::o  E:::::E
			 G:::::G       G::::Go::::o     o::::o  E:::::E       EEEEEE
			  G:::::GGGGGGGG::::Go:::::ooooo:::::oEE::::::EEEEEEEE:::::E
			   GG:::::::::::::::Go:::::::::::::::oE::::::::::::::::::::E
			     GGG::::::GGG:::G oo:::::::::::oo E::::::::::::::::::::E
			        GGGGGG   GGGG   ooooooooooo   EEEEEEEEEEEEEEEEEEEEEE

			Version 0.0.1

		`)

        //v.Editable = true

        if err := g.SetCurrentView("intro"); err != nil {
            return err
        }
	}
	return nil
}<commit_msg>Removed edible comment<commit_after>package main

import (
	"fmt"
	"../vendor/gocui"
)

func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if v, err := g.SetView("side", -1, -1, 5, maxY); err != nil {
	    if err != gocui.ErrorUnkView {
	        return err
	    }

	    fmt.Fprintln(v, "- 1")
	}

	if v, err := g.SetView("intro", 5, -1, maxX, maxY); err != nil {
        if err != gocui.ErrorUnkView {
            return err
        }
        fmt.Fprintf(v, "%s",`

			        GGGGGGGGGGGGG                 EEEEEEEEEEEEEEEEEEEEEE
			     GGG::::::::::::G                 E::::::::::::::::::::E
			   GG:::::::::::::::G                 E::::::::::::::::::::E
			  G:::::GGGGGGGG::::G                 EE::::::EEEEEEEEE::::E
			 G:::::G       GGGGGG   ooooooooooo     E:::::E       EEEEEE
			G:::::G               oo:::::::::::oo   E:::::E
			G:::::G              o:::::::::::::::o  E::::::EEEEEEEEEE
			G:::::G    GGGGGGGGGGo:::::ooooo:::::o  E:::::::::::::::E
			G:::::G    G::::::::Go::::o     o::::o  E:::::::::::::::E
			G:::::G    GGGGG::::Go::::o     o::::o  E::::::EEEEEEEEEE
			G:::::G        G::::Go::::o     o::::o  E:::::E
			 G:::::G       G::::Go::::o     o::::o  E:::::E       EEEEEE
			  G:::::GGGGGGGG::::Go:::::ooooo:::::oEE::::::EEEEEEEE:::::E
			   GG:::::::::::::::Go:::::::::::::::oE::::::::::::::::::::E
			     GGG::::::GGG:::G oo:::::::::::oo E::::::::::::::::::::E
			        GGGGGG   GGGG   ooooooooooo   EEEEEEEEEEEEEEEEEEEEEE

			Version 0.0.1

		`)

        if err := g.SetCurrentView("intro"); err != nil {
            return err
        }
	}
	return nil
}<|endoftext|>


<commit_before>package combiner

import (
  "time"
  "log"
  "fmt"
  "io/ioutil"
  "errors"
  "launchpad.net/goamz/aws"
  "launchpad.net/goamz/s3"
  "pdfcombiner/cpdf"
)

var (
  verbose = true
  bucketName = "pa-hrsuite-production"
  basedir = "/tmp/"
  keybase = "606/docs/"
  defdocs = []string{"100001.pdf","100009.pdf","100030.pdf","100035.pdf",
                     "100035.pdf","100037.pdf","100038.pdf","100093.pdf"}
)

type Job struct {
  BucketName string
  EmployerId int
  DocList    []string
  Downloaded []string
  Callback   string
  Errors     []error
  Bucket     *s3.Bucket
}

type stat struct {
  filename string
  size     int
  dlSecs   time.Duration
}

func (j *Job) IsValid() bool {
 return (j.BucketName != "") &&
        (j.Callback   != "") &&
        (j.EmployerId > 0)   &&
        (j.docCount() > 0)
}

func (j *Job) docCount() int {
  return len(j.DocList)
}

func (j *Job) markComplete(newdoc string) {
  j.Downloaded = append(j.Downloaded, newdoc)
}

func (j *Job) hasDownloadedDocs() bool {
  return len(j.Downloaded) > 0
}

func (j *Job) addError(newErr error) {
  log.Println(newErr)
  j.Errors = append(j.Errors, newErr)
}

func (j *Job) connect() {
  auth, err := aws.EnvAuth()
  if err != nil { panic(err) }
  s := s3.New(auth, aws.USEast)
  j.Bucket = s.Bucket(bucketName)
}

func (j *Job) getFile(docname string, c chan stat, e chan error) {
  start := time.Now()
  data, err := j.Bucket.Get(j.s3Path(docname))
  if err != nil {
    e <- fmt.Errorf("%v: %v", docname, err)
    return
  }
  path := "/tmp/"+docname
  err = ioutil.WriteFile(path, data, 0644)
  if err != nil {
    e <- err; return
    return
  }
  c <- stat{ filename: docname,
             size: len(data),
             dlSecs: time.Since(start) }
}

func (j *Job) s3Path(docname string) string {
  return fmt.Sprintf("%d/docs/%s", j.EmployerId, docname)
}

func (j *Job) getAllFiles() {
  start := time.Now()
  j.connect()
  c := make(chan stat)
  e := make(chan error)
  for _,doc := range j.DocList{
    go j.getFile(doc,c,e)
  }

  totalBytes := j.waitForDownloads(c,e)
  printSummary(start, totalBytes, j.docCount())
}

func (j *Job) waitForDownloads(c chan stat, e chan error) (totalBytes int) {
  for _,_ = range j.DocList{
    select {
      case packet := <-c:
        if verbose { log.Printf("%s was %d bytes\n", packet.filename,packet.size) }
        totalBytes += packet.size
        j.markComplete(packet.filename)
      case err := <-e:
        j.addError(err)
      case <-time.After(2 * time.Minute):
        j.addError(errors.New("Timed out while downloading"))
        return
    }
  }
  return
}

func printSummary(start time.Time, bytes int, count int){
  elapsed := time.Since(start)
  seconds := elapsed.Seconds()
  mbps := float64(bytes) / 1024 / 1024 / seconds
  log.Printf("got %d bytes over %d files in %f secs (%f MB/s)\n",
             bytes, count, seconds, mbps)
}

func (j *Job) postToCallback(){
  log.Println("work complete, posting success to callback:",j.Callback)
  log.Println(j)
}

func (j *Job) Combine() bool {
  defer j.postToCallback()
  j.getAllFiles()
  if j.hasDownloadedDocs() {
    cpdf.Merge(j.Downloaded)
  }
  return true
}

<commit_msg>Exported methods go at the top<commit_after>package combiner

import (
  "time"
  "log"
  "fmt"
  "io/ioutil"
  "errors"
  "launchpad.net/goamz/aws"
  "launchpad.net/goamz/s3"
  "pdfcombiner/cpdf"
)

var (
  verbose = true
  bucketName = "pa-hrsuite-production"
  basedir = "/tmp/"
  keybase = "606/docs/"
  defdocs = []string{"100001.pdf","100009.pdf","100030.pdf","100035.pdf",
                     "100035.pdf","100037.pdf","100038.pdf","100093.pdf"}
)

type Job struct {
  BucketName string
  EmployerId int
  DocList    []string
  Downloaded []string
  Callback   string
  Errors     []error
  Bucket     *s3.Bucket
}

type stat struct {
  filename string
  size     int
  dlSecs   time.Duration
}

func (j *Job) Combine() bool {
  defer j.postToCallback()
  j.getAllFiles()
  if j.hasDownloadedDocs() {
    cpdf.Merge(j.Downloaded)
  }
  return true
}

func (j *Job) IsValid() bool {
 return (j.BucketName != "") &&
        (j.Callback   != "") &&
        (j.EmployerId > 0)   &&
        (j.docCount() > 0)
}

func (j *Job) docCount() int {
  return len(j.DocList)
}

func (j *Job) markComplete(newdoc string) {
  j.Downloaded = append(j.Downloaded, newdoc)
}

func (j *Job) hasDownloadedDocs() bool {
  return len(j.Downloaded) > 0
}

func (j *Job) addError(newErr error) {
  log.Println(newErr)
  j.Errors = append(j.Errors, newErr)
}

func (j *Job) connect() {
  auth, err := aws.EnvAuth()
  if err != nil { panic(err) }
  s := s3.New(auth, aws.USEast)
  j.Bucket = s.Bucket(bucketName)
}

func (j *Job) getFile(docname string, c chan stat, e chan error) {
  start := time.Now()
  data, err := j.Bucket.Get(j.s3Path(docname))
  if err != nil {
    e <- fmt.Errorf("%v: %v", docname, err)
    return
  }
  path := "/tmp/"+docname
  err = ioutil.WriteFile(path, data, 0644)
  if err != nil {
    e <- err; return
    return
  }
  c <- stat{ filename: docname,
             size: len(data),
             dlSecs: time.Since(start) }
}

func (j *Job) s3Path(docname string) string {
  return fmt.Sprintf("%d/docs/%s", j.EmployerId, docname)
}

func (j *Job) getAllFiles() {
  start := time.Now()
  j.connect()
  c := make(chan stat)
  e := make(chan error)
  for _,doc := range j.DocList{
    go j.getFile(doc,c,e)
  }

  totalBytes := j.waitForDownloads(c,e)
  printSummary(start, totalBytes, j.docCount())
}

func (j *Job) waitForDownloads(c chan stat, e chan error) (totalBytes int) {
  for _,_ = range j.DocList{
    select {
      case packet := <-c:
        if verbose { log.Printf("%s was %d bytes\n", packet.filename,packet.size) }
        totalBytes += packet.size
        j.markComplete(packet.filename)
      case err := <-e:
        j.addError(err)
      case <-time.After(2 * time.Minute):
        j.addError(errors.New("Timed out while downloading"))
        return
    }
  }
  return
}

func printSummary(start time.Time, bytes int, count int){
  elapsed := time.Since(start)
  seconds := elapsed.Seconds()
  mbps := float64(bytes) / 1024 / 1024 / seconds
  log.Printf("got %d bytes over %d files in %f secs (%f MB/s)\n",
             bytes, count, seconds, mbps)
}

func (j *Job) postToCallback(){
  log.Println("work complete, posting success to callback:",j.Callback)
  log.Println(j)
}

<|endoftext|>


<commit_before>// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// blame: jnml, labs.nic.cz

// Package hosts supports hosts formatted data (see also `man hosts`).
// Supported are conversions from a file or string to an internal
// representation and back to a string.
package hosts

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/cznic/dns"
	"github.com/cznic/fileutil"
	"io/ioutil"
	"net"
	"os"
	"strings"
)

// System hosts name
var Sys = "/etc/hosts" //TODO:LSB only

// Cfg represents a hosts.File with a change/modification guard/handler.
type Cfg struct {
	cfg     *File
	changed bool
	mfile   *fileutil.GoMFile
	logger  *dns.Logger
}

// NewCfg returns a newly created Cfg with hosts.File initialized from file fname or an Error of any.
// It binds the contained hosts.File to fname and on any changes to the fname file the Cfg hosts.File
// is reloaded on access via Cfg.File().
func NewCfg(fname string, logger *dns.Logger) (c *Cfg, err error) {
	x := &Cfg{cfg: &File{}, logger: logger}
	if x.mfile, err = fileutil.NewGoMFile(fname, os.O_RDONLY, 0444, 0); err != nil {
		return
	}

	if err = x.cfg.Load(fname); err != nil {
		return
	}

	x.mfile.SetHandler(func(file *os.File) (err error) {
		if _, err = file.Seek(0, 0); err != nil {
			return
		}

		buf := []byte{}
		if buf, err = ioutil.ReadAll(file); err != nil {
			return
		}

		if err = x.cfg.LoadString(fname, string(buf)); err != nil {
			return
		}

		if logger.Level >= dns.LOG_EVENTS {
			logger.Log("loaded %q", fname)
			if logger.Level >= dns.LOG_DEBUG {
				logger.Log("\n%s", x.cfg)
			}
		}
		x.changed = true
		return
	})

	return x, nil
}

// SetChanged forces next File() to handle modification of the wrapped hosts.File.
func (c *Cfg) SetChanged() {
	c.mfile.SetChanged()
}

// File returns a hosts.File from Cfg and an indicator of its source file has been changed/modificated
// compared to the last invocation of this function or an Error if any.
func (c *Cfg) File() (f *File, changed bool, err error) {
	if _, err = c.mfile.File(); err != nil {
		return
	}

	f, changed, err, c.changed = c.cfg, c.changed, nil, false
	return
}

// A File holds items found in a hosts file,
// e.g. '/etc/hosts'. See also hosts(5)
type File []*FileItem

// Load File from a hosts format file fname. Return an Error, if any.
func (h *File) Load(fname string) (err error) {
	defer func() {
		if e := recover(); e != nil {
			*h = nil
			err = e.(error)
		}
	}()

	var buf []byte
	buf, err = ioutil.ReadFile(fname)
	if err != nil {
		panic(err)
	}
	return h.LoadString(fname, string(buf))
}

// Load File from a hosts format string s. Return an Error, if any.
func (h *File) LoadString(fname, s string) (err error) {
	lx := newLex(strings.NewReader(s))

	defer func() {
		if e := recover(); e != nil {
			*h = nil
			err = fmt.Errorf("%s:%d:%d - %s", fname, lx.line, lx.column, e.(error))
		}
	}()

	if yyParse(lx) != 0 {
		panic(errors.New("syntax error"))
	}

	*h = lx.hosts
	return
}

func (h *File) String() string {
	buf := bytes.NewBuffer(nil)
	for _, item := range *h {
		buf.WriteString(item.String())
		buf.WriteString("\n")
	}
	return buf.String()
}

// An FileItem holds an item found in a hosts file,
// e.g. '/etc/hosts'. See also hosts(5)
type FileItem struct {
	IP            net.IP   // Well, it's an IP address
	CanonicalName string   // Per specs canocalized hostname
	Aliases       []string // Possibly empty hostname's aliases list
}

func (i *FileItem) String() string {
	buf := bytes.NewBuffer(nil)
	buf.WriteString(fmt.Sprintf("%s %s", i.IP, i.CanonicalName))
	for _, alias := range i.Aliases {
		buf.WriteString(" ")
		buf.WriteString(alias)
	}
	return buf.String()
}

type goCfgMsg struct {
	f       *File
	changed bool
	error   error
}

// GoCfg is a Cfg wrapped in a goroutine allowing for concurrent access.
type GoCfg struct {
	cfg *Cfg
	rq  chan bool
	re  chan *goCfgMsg
}

// NewGoCfg returns a newly created GoCfg or an Error if any.
// It is assumed there will be only one/few GoCfg instance(s) in an application.
// The instance(s) and its associated goroutine(s) will never be released.
// This is by design in an atempt to avoid some possibly nasty races on finalization.
// The "impossible to release" doesn't apply when NewGoCfg returns an Error.
func NewGoCfg(fname string, logger *dns.Logger) (c *GoCfg, err error) {
	x := &GoCfg{}
	if x.cfg, err = NewCfg(fname, logger); err != nil {
		return
	}

	x.rq = make(chan bool, 10)
	x.re = make(chan *goCfgMsg, 10)
	c = x
	go func() {
		for {
			<-c.rq // wait for rq
			msg := &goCfgMsg{}
			msg.f, msg.changed, msg.error = c.cfg.File()
			c.re <- msg // return response
		}
	}()
	return
}

// File returns a File from GoCfg and an indicator if its source file has been changed/modificated
// compared to the last invocation of this function or an Error if any.
// Invocations of File are concurrent safe.
func (c *GoCfg) File() (f *File, changed bool, err error) {
	c.rq <- true
	msg := <-c.re
	return msg.f, msg.changed, msg.error
}
<commit_msg>Attempt to fix data race. Updates #4.<commit_after>// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// blame: jnml, labs.nic.cz

// Package hosts supports hosts formatted data (see also `man hosts`).
// Supported are conversions from a file or string to an internal
// representation and back to a string.
package hosts

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"sync"

	"github.com/cznic/dns"
	"github.com/cznic/fileutil"
)

// System hosts name
var Sys = "/etc/hosts" //TODO:LSB only

// Cfg represents a hosts.File with a change/modification guard/handler.
// Methods of Cfg are synchronized; multiple goroutines may invoke them
// concurrently.
type Cfg struct {
	cfg     *File
	changed bool
	mfile   *fileutil.GoMFile
	logger  *dns.Logger
	mu      sync.Mutex
}

// NewCfg returns a newly created Cfg with hosts.File initialized from file fname or an Error of any.
// It binds the contained hosts.File to fname and on any changes to the fname file the Cfg hosts.File
// is reloaded on access via Cfg.File().
func NewCfg(fname string, logger *dns.Logger) (c *Cfg, err error) {
	x := &Cfg{cfg: &File{}, logger: logger}
	if x.mfile, err = fileutil.NewGoMFile(fname, os.O_RDONLY, 0444, 0); err != nil {
		return
	}

	if err = x.cfg.Load(fname); err != nil {
		return
	}

	x.mfile.SetHandler(func(file *os.File) (err error) {
		if _, err = file.Seek(0, 0); err != nil {
			return
		}

		buf := []byte{}
		if buf, err = ioutil.ReadAll(file); err != nil {
			return
		}

		if err = x.cfg.LoadString(fname, string(buf)); err != nil {
			return
		}

		if logger.Level >= dns.LOG_EVENTS {
			logger.Log("loaded %q", fname)
			if logger.Level >= dns.LOG_DEBUG {
				logger.Log("\n%s", x.cfg)
			}
		}
		x.changed = true
		return
	})

	return x, nil
}

// SetChanged forces next File() to handle modification of the wrapped hosts.File.
func (c *Cfg) SetChanged() {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.mfile.SetChanged()
}

// File returns a hosts.File from Cfg and an indicator of its source file has been changed/modificated
// compared to the last invocation of this function or an Error if any.
func (c *Cfg) File() (f *File, changed bool, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, err = c.mfile.File(); err != nil {
		return
	}

	f, changed, err, c.changed = c.cfg, c.changed, nil, false
	return
}

// A File holds items found in a hosts file,
// e.g. '/etc/hosts'. See also hosts(5)
type File []*FileItem

// Load File from a hosts format file fname. Return an Error, if any.
func (h *File) Load(fname string) (err error) {
	defer func() {
		if e := recover(); e != nil {
			*h = nil
			err = e.(error)
		}
	}()

	var buf []byte
	buf, err = ioutil.ReadFile(fname)
	if err != nil {
		panic(err)
	}
	return h.LoadString(fname, string(buf))
}

// Load File from a hosts format string s. Return an Error, if any.
func (h *File) LoadString(fname, s string) (err error) {
	lx := newLex(strings.NewReader(s))

	defer func() {
		if e := recover(); e != nil {
			*h = nil
			err = fmt.Errorf("%s:%d:%d - %s", fname, lx.line, lx.column, e.(error))
		}
	}()

	if yyParse(lx) != 0 {
		panic(errors.New("syntax error"))
	}

	*h = lx.hosts
	return
}

func (h *File) String() string {
	buf := bytes.NewBuffer(nil)
	for _, item := range *h {
		buf.WriteString(item.String())
		buf.WriteString("\n")
	}
	return buf.String()
}

// An FileItem holds an item found in a hosts file,
// e.g. '/etc/hosts'. See also hosts(5)
type FileItem struct {
	IP            net.IP   // Well, it's an IP address
	CanonicalName string   // Per specs canocalized hostname
	Aliases       []string // Possibly empty hostname's aliases list
}

func (i *FileItem) String() string {
	buf := bytes.NewBuffer(nil)
	buf.WriteString(fmt.Sprintf("%s %s", i.IP, i.CanonicalName))
	for _, alias := range i.Aliases {
		buf.WriteString(" ")
		buf.WriteString(alias)
	}
	return buf.String()
}

type goCfgMsg struct {
	f       *File
	changed bool
	error   error
}

// GoCfg is a Cfg wrapped in a goroutine allowing for concurrent access.
type GoCfg struct {
	cfg *Cfg
	rq  chan bool
	re  chan *goCfgMsg
}

// NewGoCfg returns a newly created GoCfg or an Error if any.
// It is assumed there will be only one/few GoCfg instance(s) in an application.
// The instance(s) and its associated goroutine(s) will never be released.
// This is by design in an atempt to avoid some possibly nasty races on finalization.
// The "impossible to release" doesn't apply when NewGoCfg returns an Error.
func NewGoCfg(fname string, logger *dns.Logger) (c *GoCfg, err error) {
	x := &GoCfg{}
	if x.cfg, err = NewCfg(fname, logger); err != nil {
		return
	}

	x.rq = make(chan bool, 10)
	x.re = make(chan *goCfgMsg, 10)
	c = x
	go func() {
		for {
			<-c.rq // wait for rq
			msg := &goCfgMsg{}
			msg.f, msg.changed, msg.error = c.cfg.File()
			c.re <- msg // return response
		}
	}()
	return
}

// File returns a File from GoCfg and an indicator if its source file has been changed/modificated
// compared to the last invocation of this function or an Error if any.
// Invocations of File are concurrent safe.
func (c *GoCfg) File() (f *File, changed bool, err error) {
	c.rq <- true
	msg := <-c.re
	return msg.f, msg.changed, msg.error
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"github.com/github/git-media/gitmedia"
	"github.com/spf13/cobra"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime/debug"
	"strings"
	"time"
)

var (
	Debugging    = false
	ErrorBuffer  = &bytes.Buffer{}
	ErrorWriter  = io.MultiWriter(os.Stderr, ErrorBuffer)
	OutputWriter = io.MultiWriter(os.Stdout, ErrorBuffer)
	RootCmd      = &cobra.Command{
		Use:   "git-media",
		Short: "Git Media provides large file support to Git.",
		Run: func(cmd *cobra.Command, args []string) {
			versionCommand(cmd, args)
			cmd.Usage()
		},
	}
)

// Error prints a formatted message to Stderr.  It also gets printed to the
// panic log if one is created for this command.
func Error(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(ErrorWriter, line)
}

// Print prints a formatted message to Stdout.  It also gets printed to the
// panic log if one is created for this command.
func Print(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(OutputWriter, line)
}

// Exit prints a formatted message and exits.
func Exit(format string, args ...interface{}) {
	Error(format, args...)
	os.Exit(2)
}

// Debug prints a formatted message if debugging is enabled.  The formatted
// message also shows up in the panic log, if created.
func Debug(format string, args ...interface{}) {
	if !Debugging {
		return
	}
	log.Printf(format, args...)
}

// Panic prints a formatted message, and writes a stack trace for the error to
// a log file before exiting.
func Panic(err error, format string, args ...interface{}) {
	Error(format, args...)
	file := handlePanic(err)

	if len(file) > 0 {
		fmt.Fprintf(os.Stderr, "\nErrors logged to %s.\nUse `git media logs last` to view the log.\n", file)
	}
	os.Exit(2)
}

func Run() {
	RootCmd.Execute()
}

func PipeMediaCommand(name string, args ...string) error {
	return PipeCommand("bin/"+name, args...)
}

func PipeCommand(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func handlePanic(err error) string {
	if err == nil {
		return ""
	}

	Debug(err.Error())
	logFile, logErr := logPanic(err)
	if logErr != nil {
		fmt.Fprintf(os.Stderr, "Unable to log panic to %s - %s\n\n", gitmedia.LocalLogDir, err)
		logEnv(os.Stderr)
	}

	return logFile
}

func logEnv(w io.Writer) {
	for _, env := range gitmedia.Environ() {
		fmt.Fprintln(w, env)
	}
}

func logPanic(loggedError error) (string, error) {
	if err := os.MkdirAll(gitmedia.LocalLogDir, 0744); err != nil {
		return "", err
	}

	now := time.Now()
	name := now.Format("2006-01-02T15:04:05.999999999")
	full := filepath.Join(gitmedia.LocalLogDir, name+".log")

	file, err := os.Create(full)
	if err != nil {
		return "", err
	}

	defer file.Close()

	fmt.Fprintf(file, "> %s", filepath.Base(os.Args[0]))
	if len(os.Args) > 0 {
		fmt.Fprintf(file, " %s", strings.Join(os.Args[1:], " "))
	}
	fmt.Fprint(file, "\n")

	logEnv(file)
	fmt.Fprint(file, "\n")

	file.Write(ErrorBuffer.Bytes())
	fmt.Fprint(file, "\n")

	fmt.Fprintln(file, loggedError.Error())
	file.Write(debug.Stack())

	return full, nil
}

func init() {
	log.SetOutput(ErrorWriter)
}
<commit_msg>ラララララ ラー ララララー<commit_after>package commands

import (
	"bytes"
	"fmt"
	"github.com/github/git-media/gitmedia"
	"github.com/spf13/cobra"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime/debug"
	"strings"
	"time"
)

var (
	Debugging    = false
	ErrorBuffer  = &bytes.Buffer{}
	ErrorWriter  = io.MultiWriter(os.Stderr, ErrorBuffer)
	OutputWriter = io.MultiWriter(os.Stdout, ErrorBuffer)
	RootCmd      = &cobra.Command{
		Use:   "git-media",
		Short: "Git Media provides large file support to Git.",
		Run: func(cmd *cobra.Command, args []string) {
			versionCommand(cmd, args)
			cmd.Usage()
		},
	}
)

// Error prints a formatted message to Stderr.  It also gets printed to the
// panic log if one is created for this command.
func Error(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(ErrorWriter, line)
}

// Print prints a formatted message to Stdout.  It also gets printed to the
// panic log if one is created for this command.
func Print(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(OutputWriter, line)
}

// Exit prints a formatted message and exits.
func Exit(format string, args ...interface{}) {
	Error(format, args...)
	os.Exit(2)
}

// Debug prints a formatted message if debugging is enabled.  The formatted
// message also shows up in the panic log, if created.
func Debug(format string, args ...interface{}) {
	if !Debugging {
		return
	}
	log.Printf(format, args...)
}

// Panic prints a formatted message, and writes a stack trace for the error to
// a log file before exiting.
func Panic(err error, format string, args ...interface{}) {
	Error(format, args...)
	file := handlePanic(err)

	if len(file) > 0 {
		fmt.Fprintf(os.Stderr, "\nErrors logged to %s.\nUse `git media logs last` to view the log.\n", file)
	}
	os.Exit(2)
}

func Run() {
	RootCmd.Execute()
}

func PipeMediaCommand(name string, args ...string) error {
	return PipeCommand("bin/"+name, args...)
}

func PipeCommand(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func handlePanic(err error) string {
	if err == nil {
		return ""
	}

	Debug(err.Error())
	return logPanic(err)
}

func logEnv(w io.Writer) {
	for _, env := range gitmedia.Environ() {
		fmt.Fprintln(w, env)
	}
}

func logPanic(loggedError error) string {
	var fmtWriter io.Writer = os.Stderr

	if err := os.MkdirAll(gitmedia.LocalLogDir, 0755); err != nil {
		fmt.Fprintf(fmtWriter, "Unable to log panic to %s: %s\n\n", gitmedia.LocalLogDir, err.Error())
		return ""
	}

	now := time.Now()
	name := now.Format("2006-01-02T15:04:05.999999999")
	full := filepath.Join(gitmedia.LocalLogDir, name+".log")

	file, err := os.Create(full)
	if err != nil {
		fmt.Fprintf(fmtWriter, "Unable to log panic to %s\n\n", full)
	} else {
		fmtWriter = file
		defer file.Close()
	}

	fmt.Fprintf(fmtWriter, "> %s", filepath.Base(os.Args[0]))
	if len(os.Args) > 0 {
		fmt.Fprintf(fmtWriter, " %s", strings.Join(os.Args[1:], " "))
	}
	fmt.Fprint(fmtWriter, "\n")

	logEnv(fmtWriter)
	fmt.Fprint(fmtWriter, "\n")

	fmtWriter.Write(ErrorBuffer.Bytes())
	fmt.Fprint(fmtWriter, "\n")

	fmt.Fprintln(fmtWriter, loggedError.Error())
	fmtWriter.Write(debug.Stack())

	return full
}

func init() {
	log.SetOutput(ErrorWriter)
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/atotto/clipboard"
	"github.com/codegangsta/cli"

	"github.com/gummiboll/forgetful/reader"
	"github.com/gummiboll/forgetful/storage"
	"github.com/gummiboll/forgetful/writer"
)

// NoteName returns name of note and error if note isnt present
func NoteName(c *cli.Context) (n string, err error) {
	if c.Args().Present() != true {
		return "", errors.New("Missing argument: name")
	}

	return strings.Join(c.Args(), " "), nil
}

// AddCommand adds a Note
func AddCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	if exists := i.NoteExists(nName); exists == true {
		return n, fmt.Errorf("Note already exists")
	}

	n.Name = nName
	n.Temporary = c.Bool("t")

	// Only open editor if -p (read from clipboard) isnt set
	if c.IsSet("p") {
		nText, err := clipboard.ReadAll()
		if err != nil {
			return n, err
		}
		n.Text = nText
	} else {
		if err := writer.WriteNote(&n); err != nil {
			return n, err
		}
	}

	if err := i.SaveNote(&n); err != nil {
		return n, err
	}

	return n, nil
}

// DeleteCommand deletes a Note
func DeleteCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, err
	}

	if i.DeleteNote(n) != nil {
		return n, err
	}

	return n, nil
}

// EditCommand edits a Note
func EditCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, err
	}

	if err := writer.WriteNote(&n); err != nil {
		return n, err
	}

	if err := i.SaveNote(&n); err != nil {
		return n, err
	}

	return n, nil

}

// ReadCommand reads a Note
func ReadCommand(c *cli.Context, i storage.Impl) (err error) {
	nName, err := NoteName(c)
	if err != nil {
		return err
	}

	n, err := i.LoadNote(nName)
	if err != nil {
		return err
	}

	if err := reader.ReadNote(n); err != nil {
		return err
	}

	return nil
}

// ListCommand lists Notes
func ListCommand(c *cli.Context, i storage.Impl) (rnotes []string) {
	nName := strings.Join(c.Args(), " ")
	notes := i.ListNotes(nName)

	for _, n := range notes {
		nStr := fmt.Sprintf("\u2023 %s", n.Name)
		if n.Temporary {
			validTo := n.CreatedAt.Add(24 * time.Hour)
			dur := validTo.Sub(time.Now())
			nStr += fmt.Sprintf(" (valid for %s)", dur)
		}
		rnotes = append(rnotes, nStr)
	}

	return rnotes

}

// SearchCommand searches for Notes
func SearchCommand(c *cli.Context, i storage.Impl) (rnotes []string, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return rnotes, err
	}

	notes := i.SearchNotes(nName)

	for _, n := range notes {
		nStr := fmt.Sprintf("\u2023 %s", n.Name)
		if n.Temporary {
			validTo := n.CreatedAt.Add(24 * time.Hour)
			dur := validTo.Sub(time.Now())
			nStr += fmt.Sprintf(" (valid for %s)", dur)
		}

		rnotes = append(rnotes, nStr)
	}

	return rnotes, nil
}

// ShareCommand shares a Note
func ShareCommand(c *cli.Context, i storage.Impl) (n storage.Note, url string, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, url, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, url, err
	}

	url, err = reader.ShareNote(n, "http://hastebin.com/documents")

	if err != nil {
		return n, url, err
	}

	return n, url, nil
}

// KeepCommand keeps a Note
func KeepCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, err
	}

	n.Temporary = false
	if err := i.SaveNote(&n); err != nil {
		return n, err
	}

	return n, err
}
<commit_msg>Sorts list/search results<commit_after>package commands

import (
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/atotto/clipboard"
	"github.com/codegangsta/cli"

	"github.com/gummiboll/forgetful/reader"
	"github.com/gummiboll/forgetful/storage"
	"github.com/gummiboll/forgetful/writer"
)

type byLetterNocase []string

func (s byLetterNocase) Len() int {
	return len(s)
}
func (s byLetterNocase) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s byLetterNocase) Less(i, j int) bool {
	return strings.ToLower(s[i]) < strings.ToLower(s[j])
}

// FormatNoteList returns a formatted list of notes
func FormatNoteList(notes []storage.Note) (rnotes []string) {
	for _, n := range notes {
		nStr := fmt.Sprintf("\u2023 %s", n.Name)
		if n.Temporary {
			validTo := n.CreatedAt.Add(24 * time.Hour)
			dur := validTo.Sub(time.Now())
			nStr += fmt.Sprintf(" (valid for %s)", dur)
		}
		rnotes = append(rnotes, nStr)
	}

	sort.Sort(byLetterNocase(rnotes))

	return rnotes
}

// NoteName returns name of note and error if note isnt present
func NoteName(c *cli.Context) (n string, err error) {
	if c.Args().Present() != true {
		return "", errors.New("Missing argument: name")
	}

	return strings.Join(c.Args(), " "), nil
}

// AddCommand adds a Note
func AddCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	if exists := i.NoteExists(nName); exists == true {
		return n, fmt.Errorf("Note already exists")
	}

	n.Name = nName
	n.Temporary = c.Bool("t")

	// Only open editor if -p (read from clipboard) isnt set
	if c.IsSet("p") {
		nText, err := clipboard.ReadAll()
		if err != nil {
			return n, err
		}
		n.Text = nText
	} else {
		if err := writer.WriteNote(&n); err != nil {
			return n, err
		}
	}

	if err := i.SaveNote(&n); err != nil {
		return n, err
	}

	return n, nil
}

// DeleteCommand deletes a Note
func DeleteCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, err
	}

	if i.DeleteNote(n) != nil {
		return n, err
	}

	return n, nil
}

// EditCommand edits a Note
func EditCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, err
	}

	if err := writer.WriteNote(&n); err != nil {
		return n, err
	}

	if err := i.SaveNote(&n); err != nil {
		return n, err
	}

	return n, nil

}

// ReadCommand reads a Note
func ReadCommand(c *cli.Context, i storage.Impl) (err error) {
	nName, err := NoteName(c)
	if err != nil {
		return err
	}

	n, err := i.LoadNote(nName)
	if err != nil {
		return err
	}

	if err := reader.ReadNote(n); err != nil {
		return err
	}

	return nil
}

// ListCommand lists Notes
func ListCommand(c *cli.Context, i storage.Impl) (rnotes []string) {
	nName := strings.Join(c.Args(), " ")
	notes := i.ListNotes(nName)

	return FormatNoteList(notes)

}

// SearchCommand searches for Notes
func SearchCommand(c *cli.Context, i storage.Impl) (rnotes []string, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return rnotes, err
	}

	notes := i.SearchNotes(nName)

	return FormatNoteList(notes), nil
}

// ShareCommand shares a Note
func ShareCommand(c *cli.Context, i storage.Impl) (n storage.Note, url string, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, url, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, url, err
	}

	url, err = reader.ShareNote(n, "http://hastebin.com/documents")

	if err != nil {
		return n, url, err
	}

	return n, url, nil
}

// KeepCommand keeps a Note
func KeepCommand(c *cli.Context, i storage.Impl) (n storage.Note, err error) {
	nName, err := NoteName(c)
	if err != nil {
		return n, err
	}

	n, err = i.LoadNote(nName)
	if err != nil {
		return n, err
	}

	n.Temporary = false
	if err := i.SaveNote(&n); err != nil {
		return n, err
	}

	return n, err
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/khlieng/dispatch/assets"
	"github.com/khlieng/dispatch/server"
	"github.com/khlieng/dispatch/storage"
)

const logo = `
    ____   _                     _         _
   |  _ \ (_) ___  _ __    __ _ | |_  ___ | |__
   | | | || |/ __|| '_ \  / _  || __|/ __|| '_ \
   | |_| || |\__ \| |_) || (_| || |_| (__ | | | |
   |____/ |_||___/| .__/  \__,_| \__|\___||_| |_|
                  |_|
                       v0.4
`

var rootCmd = &cobra.Command{
	Use:   "dispatch",
	Short: "Web-based IRC client in Go.",
	PersistentPreRun: func(cmd *cobra.Command, args []string) {
		if cmd.Use == "dispatch" {
			fmt.Println(logo)
		}

		storage.Initialize(viper.GetString("dir"))

		initConfig(storage.Path.Config())

		viper.SetConfigName("config")
		viper.AddConfigPath(storage.Path.Root())
		viper.ReadInConfig()

		viper.WatchConfig()

		prev := time.Now()
		viper.OnConfigChange(func(e fsnotify.Event) {
			now := time.Now()
			// fsnotify sometimes fires twice
			if now.Sub(prev) > time.Second {
				log.Println("New config loaded")
				prev = now
			}
		})
	},

	Run: func(cmd *cobra.Command, args []string) {
		if viper.GetBool("dev") {
			log.Println("Running in development mode, access client at http://localhost:3000")
		}
		log.Println("Storing data at", storage.Path.Root())

		storage.Open()
		defer storage.Close()

		server.Run()
	},
}

func Execute() {
	rootCmd.Execute()
}

func init() {
	rootCmd.AddCommand(clearCmd)
	rootCmd.AddCommand(configCmd)

	rootCmd.PersistentFlags().String("dir", storage.DefaultDirectory(), "directory to store config and data in")
	rootCmd.Flags().IntP("port", "p", 80, "port to listen on")
	rootCmd.Flags().Bool("dev", false, "development mode")

	viper.BindPFlag("dir", rootCmd.PersistentFlags().Lookup("dir"))
	viper.BindPFlag("port", rootCmd.Flags().Lookup("port"))
	viper.BindPFlag("dev", rootCmd.Flags().Lookup("dev"))

	viper.SetDefault("verify_client_certificates", true)
}

func initConfig(configPath string) {
	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		config, err := assets.Asset("config.default.toml")
		if err != nil {
			log.Println(err)
			return
		}

		log.Println("Writing default config to", configPath)

		err = ioutil.WriteFile(configPath, config, 0600)
		if err != nil {
			log.Println(err)
		}
	}
}
<commit_msg>Add reset-config flag<commit_after>package commands

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/khlieng/dispatch/assets"
	"github.com/khlieng/dispatch/server"
	"github.com/khlieng/dispatch/storage"
)

const logo = `
    ____   _                     _         _
   |  _ \ (_) ___  _ __    __ _ | |_  ___ | |__
   | | | || |/ __|| '_ \  / _  || __|/ __|| '_ \
   | |_| || |\__ \| |_) || (_| || |_| (__ | | | |
   |____/ |_||___/| .__/  \__,_| \__|\___||_| |_|
                  |_|
                       v0.4
`

var rootCmd = &cobra.Command{
	Use:   "dispatch",
	Short: "Web-based IRC client in Go.",
	PersistentPreRun: func(cmd *cobra.Command, args []string) {
		if cmd.Use == "dispatch" {
			fmt.Println(logo)
		}

		storage.Initialize(viper.GetString("dir"))

		initConfig(storage.Path.Config(), viper.GetBool("reset_config"))

		viper.SetConfigName("config")
		viper.AddConfigPath(storage.Path.Root())
		viper.ReadInConfig()

		viper.WatchConfig()

		prev := time.Now()
		viper.OnConfigChange(func(e fsnotify.Event) {
			now := time.Now()
			// fsnotify sometimes fires twice
			if now.Sub(prev) > time.Second {
				log.Println("New config loaded")
				prev = now
			}
		})
	},

	Run: func(cmd *cobra.Command, args []string) {
		if viper.GetBool("dev") {
			log.Println("Running in development mode, access client at http://localhost:3000")
		}
		log.Println("Storing data at", storage.Path.Root())

		storage.Open()
		defer storage.Close()

		server.Run()
	},
}

func Execute() {
	rootCmd.Execute()
}

func init() {
	rootCmd.AddCommand(clearCmd)
	rootCmd.AddCommand(configCmd)

	rootCmd.PersistentFlags().String("dir", storage.DefaultDirectory(), "directory to store config and data in")
	rootCmd.PersistentFlags().Bool("reset-config", false, "reset to the default configuration, overwriting the current one")
	rootCmd.Flags().IntP("port", "p", 80, "port to listen on")
	rootCmd.Flags().Bool("dev", false, "development mode")

	viper.BindPFlag("dir", rootCmd.PersistentFlags().Lookup("dir"))
	viper.BindPFlag("reset_config", rootCmd.PersistentFlags().Lookup("reset-config"))
	viper.BindPFlag("port", rootCmd.Flags().Lookup("port"))
	viper.BindPFlag("dev", rootCmd.Flags().Lookup("dev"))

	viper.SetDefault("verify_client_certificates", true)
}

func initConfig(configPath string, overwrite bool) {
	if _, err := os.Stat(configPath); overwrite || os.IsNotExist(err) {
		config, err := assets.Asset("config.default.toml")
		if err != nil {
			log.Println(err)
			return
		}

		log.Println("Writing default config to", configPath)

		err = ioutil.WriteFile(configPath, config, 0600)
		if err != nil {
			log.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"time"

	"github.com/fabric8io/kadvisor/utils"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var KadvisorCmd = &cobra.Command{
	Use:   "kadvisor",
	Short: "KAdvisor is a metrics collector & publisher for Kubernetes",
	Long:  "A configurable metrics collector & publisher for Kubernetes",
	Run: func(cmd *cobra.Command, args []string) {
		InitializeConfig()
	},
}

var kadvisorCmdV *cobra.Command

var (
	VerboseLog   bool
	CfgFile      string
	PollDuration time.Duration
)

func Execute() {
	utils.StopOnErr(KadvisorCmd.Execute())
}

//Initializes flags
func init() {
	KadvisorCmd.PersistentFlags().StringVarP(&CfgFile, "config", "c", "kadvisor.yml", "config file")
	KadvisorCmd.PersistentFlags().BoolVarP(&VerboseLog, "verbose", "v", false, "verbose logging")
	KadvisorCmd.PersistentFlags().DurationVarP(&PollDuration, "poll", "p", 10*time.Second, "poll duration")
	kadvisorCmdV = KadvisorCmd
}

// InitializeConfig initializes a config file with sensible default configuration flags.
func InitializeConfig() {
	viper.SetConfigFile(CfgFile)
}
<commit_msg>Better configuration & more logging<commit_after>package commands

import (
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/fabric8io/kadvisor/utils"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var KadvisorCmd = &cobra.Command{
	Use:   "kadvisor",
	Short: "KAdvisor is a metrics collector & publisher for Kubernetes",
	Long:  "A configurable metrics collector & publisher for Kubernetes",
	Run: func(cmd *cobra.Command, args []string) {
		InitializeConfig()
	},
}

var kadvisorCmdV *cobra.Command

var (
	Verbose      bool
	CfgFile      string
	PollDuration time.Duration
)

func Execute() {
	utils.StopOnErr(KadvisorCmd.Execute())
}

//Initializes flags
func init() {
	KadvisorCmd.PersistentFlags().StringVarP(&CfgFile, "config", "c", "", "config file")
	KadvisorCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose logging")
	KadvisorCmd.PersistentFlags().DurationVarP(&PollDuration, "poll", "p", 10*time.Second, "poll duration")
	kadvisorCmdV = KadvisorCmd
}

// InitializeConfig initializes a config file with sensible default configuration flags.
func InitializeConfig() {
	if len(CfgFile) > 0 {
		viper.SetConfigFile(CfgFile)
	} else {
		viper.SetConfigName("config")
		viper.AddConfigPath("$HOME/.kadvisor")
		viper.AddConfigPath("/etc/kadvisor")
		viper.AddConfigPath(".")
	}

	viper.SetEnvPrefix("kadvisor")
	viper.AutomaticEnv()

	err := viper.ReadInConfig()
	if err != nil {
		log.Debug("Unable to locate Config file")
	}

	viper.SetDefault("verbose", false)
	viper.SetDefault("poll", 10*time.Second)

	if kadvisorCmdV.PersistentFlags().Lookup("verbose").Changed {
		viper.Set("verbose", Verbose)
	}
	if kadvisorCmdV.PersistentFlags().Lookup("poll").Changed {
		viper.Set("poll", PollDuration)
	}

	if viper.GetBool("verbose") {
		log.SetLevel(log.DebugLevel)
	}

	log.WithField("config", viper.AllSettings()).Debug("Configured settings")
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package model

import (
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"encoding/base32"
	"encoding/json"
	"fmt"
	"io"
	"net/mail"
	"regexp"
	"strings"
	"time"
)

const (
	ETAG_ROOT_VERSION = "10"
)

type StringMap map[string]string
type StringArray []string
type EncryptStringMap map[string]string

// AppError is returned for any http response that's not in the 200 range.
type AppError struct {
	Message       string `json:"message"`        // Message to be display to the end user without debugging information
	DetailedError string `json:"detailed_error"` // Internal error string to help the developer
	RequestId     string `json:"request_id"`     // The RequestId that's also set in the header
	StatusCode    int    `json:"status_code"`    // The http status code
	Where         string `json:"-"`              // The function where it happened in the form of Struct.Func
}

func (er *AppError) Error() string {
	return er.Where + ": " + er.Message + ", " + er.DetailedError
}

func (er *AppError) ToJson() string {
	b, err := json.Marshal(er)
	if err != nil {
		return ""
	} else {
		return string(b)
	}
}

// AppErrorFromJson will decode the input and return an AppError
func AppErrorFromJson(data io.Reader) *AppError {
	decoder := json.NewDecoder(data)
	var er AppError
	err := decoder.Decode(&er)
	if err == nil {
		return &er
	} else {
		return nil
	}
}

func NewAppError(where string, message string, details string) *AppError {
	ap := &AppError{}
	ap.Message = message
	ap.Where = where
	ap.DetailedError = details
	ap.StatusCode = 500
	return ap
}

var encoding = base32.NewEncoding("ybndrfg8ejkmcpqxot1uwisza345h769")

// NewId is a globally unique identifier.  It is a [A-Z0-9] string 26
// characters long.  It is a UUID version 4 Guid that is zbased32 encoded
// with the padding stripped off.
func NewId() string {
	var b bytes.Buffer
	encoder := base32.NewEncoder(encoding, &b)
	encoder.Write(uuid.NewRandom())
	encoder.Close()
	b.Truncate(26) // removes the '==' padding
	return b.String()
}

// GetMillis is a convience method to get milliseconds since epoch.
func GetMillis() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

// MapToJson converts a map to a json string
func MapToJson(objmap map[string]string) string {
	if b, err := json.Marshal(objmap); err != nil {
		return ""
	} else {
		return string(b)
	}
}

// MapFromJson will decode the key/value pair map
func MapFromJson(data io.Reader) map[string]string {
	decoder := json.NewDecoder(data)

	var objmap map[string]string
	if err := decoder.Decode(&objmap); err != nil {
		return make(map[string]string)
	} else {
		return objmap
	}
}

func ArrayToJson(objmap []string) string {
	if b, err := json.Marshal(objmap); err != nil {
		return ""
	} else {
		return string(b)
	}
}

func ArrayFromJson(data io.Reader) []string {
	decoder := json.NewDecoder(data)

	var objmap []string
	if err := decoder.Decode(&objmap); err != nil {
		return make([]string, 0)
	} else {
		return objmap
	}
}

func IsLower(s string) bool {
	if strings.ToLower(s) == s {
		return true
	}

	return false
}

func IsValidEmail(email string) bool {

	if !IsLower(email) {
		return false
	}

	if _, err := mail.ParseAddress(email); err == nil {
		return true
	}

	return false
}

var reservedDomains = []string{
	"www",
	"web",
	"admin",
	"support",
	"notify",
	"test",
	"demo",
	"mail",
	"team",
	"channel",
	"internal",
	"localhost",
	"dockerhost",
	"stag",
	"post",
	"cluster",
	"api",
}

func IsReservedDomain(s string) bool {
	s = strings.ToLower(s)

	for _, value := range reservedDomains {
		if strings.Index(s, value) == 0 {
			return true
		}
	}

	return false
}

func IsValidDomain(s string) bool {

	if !IsValidAlphaNum(s) {
		return false
	}

	if len(s) <= 3 {
		return false
	}

	return true
}

var wwwStart = regexp.MustCompile(`^www`)
var betaStart = regexp.MustCompile(`^beta`)
var ciStart = regexp.MustCompile(`^ci`)

func GetSubDomain(s string) (string, string) {
	s = strings.Replace(s, "http://", "", 1)
	s = strings.Replace(s, "https://", "", 1)

	match := wwwStart.MatchString(s)
	if match {
		return "", ""
	}

	match = betaStart.MatchString(s)
	if match {
		return "", ""
	}

	match = ciStart.MatchString(s)
	if match {
		return "", ""
	}

	parts := strings.Split(s, ".")

	if len(parts) != 3 {
		return "", ""
	}

	return parts[0], parts[1]
}

func IsValidChannelIdentifier(s string) bool {

	if !IsValidAlphaNum(s) {
		return false
	}

	if len(s) < 2 {
		return false
	}

	return true
}

var validAlphaNum = regexp.MustCompile(`^[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+$`)

func IsValidAlphaNum(s string) bool {
	match := validAlphaNum.MatchString(s)

	if !match {
		return false
	}

	return true
}

func Etag(parts ...interface{}) string {

	etag := ETAG_ROOT_VERSION

	for _, part := range parts {
		etag += fmt.Sprintf(".%v", part)
	}

	return etag
}

var validHashtag = regexp.MustCompile(`^(#[A-Za-z]+[A-Za-z0-9_]*[A-Za-z0-9])$`)
var puncStart = regexp.MustCompile(`^[.,()&$!\[\]{}"':;\\]+`)
var puncEnd = regexp.MustCompile(`[.,()&$#!\[\]{}"':;\\]+$`)

func ParseHashtags(text string) (string, string) {
	words := strings.Split(strings.Replace(text, "\n", " ", -1), " ")

	hashtagString := ""
	plainString := ""
	for _, word := range words {
		word = puncStart.ReplaceAllString(word, "")
		word = puncEnd.ReplaceAllString(word, "")
		if validHashtag.MatchString(word) {
			hashtagString += " " + word
		} else {
			plainString += " " + word
		}
	}

	if len(hashtagString) > 1000 {
		hashtagString = hashtagString[:999]
		lastSpace := strings.LastIndex(hashtagString, " ")
		if lastSpace > -1 {
			hashtagString = hashtagString[:lastSpace]
		} else {
			hashtagString = ""
		}
	}

	return strings.TrimSpace(hashtagString), strings.TrimSpace(plainString)
}

func IsFileExtImage(ext string) bool {
	ext = strings.ToLower(ext)
	for _, imgExt := range IMAGE_EXTENSIONS {
		if ext == imgExt {
			return true
		}
	}
	return false
}

func GetImageMimeType(ext string) string {
	ext = strings.ToLower(ext)
	if len(IMAGE_MIME_TYPES[ext]) == 0 {
		return "image"
	} else {
		return IMAGE_MIME_TYPES[ext]
	}
}

func ClearMentionTags(post string) string {
	post = strings.Replace(post, "<mention>", "", -1)
	post = strings.Replace(post, "</mention>", "", -1)
	return post
}

var UrlRegex = regexp.MustCompile(`^((?:[a-z]+:\/\/)?(?:(?:[a-z0-9\-]+\.)+(?:[a-z]{2}|aero|arpa|biz|com|coop|edu|gov|info|int|jobs|mil|museum|name|nato|net|org|pro|travel|local|internal))(:[0-9]{1,5})?(?:\/[a-z0-9_\-\.~]+)*(\/([a-z0-9_\-\.]*)(?:\?[a-z0-9+_~\-\.%=&]*)?)?(?:#[a-zA-Z0-9!$&'()*+.=-_~:@/?]*)?)(?:\s+|$)$`)
var PartialUrlRegex = regexp.MustCompile(`/api/v1/files/(get|get_image)/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/(([A-Za-z0-9]+/)?.+\.[A-Za-z0-9]{3,})`)

var SplitRunes = map[rune]bool{',': true, ' ': true, '.': true, '!': true, '?': true, ':': true, ';': true, '\n': true, '<': true, '>': true, '(': true, ')': true, '{': true, '}': true, '[': true, ']': true, '+': true, '/': true, '\\': true}
<commit_msg>update etag version<commit_after>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package model

import (
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"encoding/base32"
	"encoding/json"
	"fmt"
	"io"
	"net/mail"
	"regexp"
	"strings"
	"time"
)

const (
	ETAG_ROOT_VERSION = "11"
)

type StringMap map[string]string
type StringArray []string
type EncryptStringMap map[string]string

// AppError is returned for any http response that's not in the 200 range.
type AppError struct {
	Message       string `json:"message"`        // Message to be display to the end user without debugging information
	DetailedError string `json:"detailed_error"` // Internal error string to help the developer
	RequestId     string `json:"request_id"`     // The RequestId that's also set in the header
	StatusCode    int    `json:"status_code"`    // The http status code
	Where         string `json:"-"`              // The function where it happened in the form of Struct.Func
}

func (er *AppError) Error() string {
	return er.Where + ": " + er.Message + ", " + er.DetailedError
}

func (er *AppError) ToJson() string {
	b, err := json.Marshal(er)
	if err != nil {
		return ""
	} else {
		return string(b)
	}
}

// AppErrorFromJson will decode the input and return an AppError
func AppErrorFromJson(data io.Reader) *AppError {
	decoder := json.NewDecoder(data)
	var er AppError
	err := decoder.Decode(&er)
	if err == nil {
		return &er
	} else {
		return nil
	}
}

func NewAppError(where string, message string, details string) *AppError {
	ap := &AppError{}
	ap.Message = message
	ap.Where = where
	ap.DetailedError = details
	ap.StatusCode = 500
	return ap
}

var encoding = base32.NewEncoding("ybndrfg8ejkmcpqxot1uwisza345h769")

// NewId is a globally unique identifier.  It is a [A-Z0-9] string 26
// characters long.  It is a UUID version 4 Guid that is zbased32 encoded
// with the padding stripped off.
func NewId() string {
	var b bytes.Buffer
	encoder := base32.NewEncoder(encoding, &b)
	encoder.Write(uuid.NewRandom())
	encoder.Close()
	b.Truncate(26) // removes the '==' padding
	return b.String()
}

// GetMillis is a convience method to get milliseconds since epoch.
func GetMillis() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

// MapToJson converts a map to a json string
func MapToJson(objmap map[string]string) string {
	if b, err := json.Marshal(objmap); err != nil {
		return ""
	} else {
		return string(b)
	}
}

// MapFromJson will decode the key/value pair map
func MapFromJson(data io.Reader) map[string]string {
	decoder := json.NewDecoder(data)

	var objmap map[string]string
	if err := decoder.Decode(&objmap); err != nil {
		return make(map[string]string)
	} else {
		return objmap
	}
}

func ArrayToJson(objmap []string) string {
	if b, err := json.Marshal(objmap); err != nil {
		return ""
	} else {
		return string(b)
	}
}

func ArrayFromJson(data io.Reader) []string {
	decoder := json.NewDecoder(data)

	var objmap []string
	if err := decoder.Decode(&objmap); err != nil {
		return make([]string, 0)
	} else {
		return objmap
	}
}

func IsLower(s string) bool {
	if strings.ToLower(s) == s {
		return true
	}

	return false
}

func IsValidEmail(email string) bool {

	if !IsLower(email) {
		return false
	}

	if _, err := mail.ParseAddress(email); err == nil {
		return true
	}

	return false
}

var reservedDomains = []string{
	"www",
	"web",
	"admin",
	"support",
	"notify",
	"test",
	"demo",
	"mail",
	"team",
	"channel",
	"internal",
	"localhost",
	"dockerhost",
	"stag",
	"post",
	"cluster",
	"api",
}

func IsReservedDomain(s string) bool {
	s = strings.ToLower(s)

	for _, value := range reservedDomains {
		if strings.Index(s, value) == 0 {
			return true
		}
	}

	return false
}

func IsValidDomain(s string) bool {

	if !IsValidAlphaNum(s) {
		return false
	}

	if len(s) <= 3 {
		return false
	}

	return true
}

var wwwStart = regexp.MustCompile(`^www`)
var betaStart = regexp.MustCompile(`^beta`)
var ciStart = regexp.MustCompile(`^ci`)

func GetSubDomain(s string) (string, string) {
	s = strings.Replace(s, "http://", "", 1)
	s = strings.Replace(s, "https://", "", 1)

	match := wwwStart.MatchString(s)
	if match {
		return "", ""
	}

	match = betaStart.MatchString(s)
	if match {
		return "", ""
	}

	match = ciStart.MatchString(s)
	if match {
		return "", ""
	}

	parts := strings.Split(s, ".")

	if len(parts) != 3 {
		return "", ""
	}

	return parts[0], parts[1]
}

func IsValidChannelIdentifier(s string) bool {

	if !IsValidAlphaNum(s) {
		return false
	}

	if len(s) < 2 {
		return false
	}

	return true
}

var validAlphaNum = regexp.MustCompile(`^[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+$`)

func IsValidAlphaNum(s string) bool {
	match := validAlphaNum.MatchString(s)

	if !match {
		return false
	}

	return true
}

func Etag(parts ...interface{}) string {

	etag := ETAG_ROOT_VERSION

	for _, part := range parts {
		etag += fmt.Sprintf(".%v", part)
	}

	return etag
}

var validHashtag = regexp.MustCompile(`^(#[A-Za-z]+[A-Za-z0-9_]*[A-Za-z0-9])$`)
var puncStart = regexp.MustCompile(`^[.,()&$!\[\]{}"':;\\]+`)
var puncEnd = regexp.MustCompile(`[.,()&$#!\[\]{}"':;\\]+$`)

func ParseHashtags(text string) (string, string) {
	words := strings.Split(strings.Replace(text, "\n", " ", -1), " ")

	hashtagString := ""
	plainString := ""
	for _, word := range words {
		word = puncStart.ReplaceAllString(word, "")
		word = puncEnd.ReplaceAllString(word, "")
		if validHashtag.MatchString(word) {
			hashtagString += " " + word
		} else {
			plainString += " " + word
		}
	}

	if len(hashtagString) > 1000 {
		hashtagString = hashtagString[:999]
		lastSpace := strings.LastIndex(hashtagString, " ")
		if lastSpace > -1 {
			hashtagString = hashtagString[:lastSpace]
		} else {
			hashtagString = ""
		}
	}

	return strings.TrimSpace(hashtagString), strings.TrimSpace(plainString)
}

func IsFileExtImage(ext string) bool {
	ext = strings.ToLower(ext)
	for _, imgExt := range IMAGE_EXTENSIONS {
		if ext == imgExt {
			return true
		}
	}
	return false
}

func GetImageMimeType(ext string) string {
	ext = strings.ToLower(ext)
	if len(IMAGE_MIME_TYPES[ext]) == 0 {
		return "image"
	} else {
		return IMAGE_MIME_TYPES[ext]
	}
}

func ClearMentionTags(post string) string {
	post = strings.Replace(post, "<mention>", "", -1)
	post = strings.Replace(post, "</mention>", "", -1)
	return post
}

var UrlRegex = regexp.MustCompile(`^((?:[a-z]+:\/\/)?(?:(?:[a-z0-9\-]+\.)+(?:[a-z]{2}|aero|arpa|biz|com|coop|edu|gov|info|int|jobs|mil|museum|name|nato|net|org|pro|travel|local|internal))(:[0-9]{1,5})?(?:\/[a-z0-9_\-\.~]+)*(\/([a-z0-9_\-\.]*)(?:\?[a-z0-9+_~\-\.%=&]*)?)?(?:#[a-zA-Z0-9!$&'()*+.=-_~:@/?]*)?)(?:\s+|$)$`)
var PartialUrlRegex = regexp.MustCompile(`/api/v1/files/(get|get_image)/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/(([A-Za-z0-9]+/)?.+\.[A-Za-z0-9]{3,})`)

var SplitRunes = map[rune]bool{',': true, ' ': true, '.': true, '!': true, '?': true, ':': true, ';': true, '\n': true, '<': true, '>': true, '(': true, ')': true, '{': true, '}': true, '[': true, ']': true, '+': true, '/': true, '\\': true}
<|endoftext|>