<commit_before>// Copyright 2015 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// validhtml.go contains checks to slighty validate a HTML document.

package ht

import (
	"fmt"
	"io"
	"net/url"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/text/language"
)

func init() {
	RegisterCheck(ValidHTML{})
}

// ValidHTML checks for valid HTML 5; well kinda: It make sure that some
// common but easy to detect fuckups are not present. The following issues
// are detected:
//   * 'doctype':   not exactly one DOCTYPE
//   * 'structure': ill-formed tag nesting / tag closing
//   * 'uniqueids': uniqness of id attribute values
//   * 'lang':      ill-formed lang attributes
//   * 'attr':      dupplicate attributes
//   * 'escaping':  unescaped &, < and > characters or unknown entities
//   * 'label':     reference to nonexisting id in label tags
//   * 'url':       malformed URLs
type ValidHTML struct {
	// Ignore is a space seperated list of issues to ignore.
	// You normaly won't skip detection of these issues as all issues
	// are fundamental flaw which are easy to fix.
	Ignore string `json:",omitempty"`
}

// Execute implements Check's Execute method.
func (v ValidHTML) Execute(t *Test) error {
	if t.Response.BodyErr != nil {
		return ErrBadBody
	}

	mask, _ := ignoreMask(v.Ignore)
	state := newHtmlState(t.Response.BodyStr, mask)

	// Parse document and record local errors in state.
	z := html.NewTokenizer(state)
	depth := 0
done:
	for {
		tt := z.Next()
		// fmt.Printf("%s%s: ", strings.Repeat("  ", depth), tt)
		switch tt {
		case html.ErrorToken:
			if z.Err() == io.EOF {
				break done
			}
			return z.Err()
		case html.TextToken:
			// fmt.Printf(" %q\n", z.Text())
			// fmt.Println()
			if depth > 0 {
				state.checkEscaping(string(z.Raw()))
			}
		case html.StartTagToken, html.SelfClosingTagToken:
			raw := string(z.Raw())
			if len(raw) > 3 {
				state.checkEscaping(raw[1 : len(raw)-1])
			}
			tn, hasAttr := z.TagName()
			// Some tags are empty and may be written in the self-closing
			// variant "
" or simply empty like "
".
			// TODO: Maybe allow the non-compliant form "

" too?
			if tt != html.SelfClosingTagToken && !emptyHTMLElement[string(tn)] {
				state.push(string(tn))
				depth++
			}
			tag := string(tn)
			// fmt.Printf(" %s  ", tag)
			state.count(tag)
			attrs := map[string]string{}
			var bkey, bval []byte
			for hasAttr {
				bkey, bval, hasAttr = z.TagAttr()
				key, val := string(bkey), string(bval)
				// fmt.Printf("%s=%s ", key, val)
				if _, ok := attrs[key]; ok {
					if state.ignore&issueAttr == 0 {
						state.err(fmt.Errorf("duplicate attribute '%s'", key))
					}
				}
				attrs[key] = val
				switch {
				case key == "id":
					state.checkID(val)
				case tag == "label" && key == "for":
					state.recordLabel(val)
				case key == "lang":
					state.checkLang(val)
				case (tag == "a" && key == "href") ||
					(tag == "img" && key == "src"): // TODO link?
					state.checkURL(val)
				}
			}
			// fmt.Println()
		case html.EndTagToken:
			tn, _ := z.TagName()
			// fmt.Println(" ", string(tn))
			state.pop(string(tn))
			depth--
		case html.CommentToken:
		case html.DoctypeToken:
			state.count("DOCTYPE")
		}
	}

	// Check for global errors.
	state.line++ // Global errors are reported "after the last line".
	if state.ignore&issueDoctype == 0 {
		if d := state.elementCount["DOCTYPE"]; d != 1 {
			state.err(fmt.Errorf("found %d DOCTYPE", d))
		}
	}
	if state.ignore&issueLabelRef == 0 {
		for _, id := range state.labelFor {
			if _, ok := state.seenIDs[id]; !ok {
				state.err(fmt.Errorf("label references unknown id '%s'", id))
			}
		}
	}

	if len(state.errors) == 0 {
		return nil
	}

	if len(state.errors) == 0 {
		return nil
	}
	return state.errors
}

// Prepare implements Check's Prepare method.
func (v ValidHTML) Prepare() error {
	_, err := ignoreMask(v.Ignore)
	return err
}

// emptyHTMLElement is the list of all HTML5 elements which are empty, that
// is they are implecitely self-closing and can be written either in
// XML-style like e.g. "
" or in HTML5-style just "
".
// The list was taken from:
//   - http://www.elharo.com/blog/software-development/web-development/2007/01/29/all-empty-tags-in-html/
// Someone should check the whole list here: http://www.w3schools.com/tags/default.asp
var emptyHTMLElement = map[string]bool{
	"br":    true,
	"hr":    true,
	"meta":  true,
	"base":  true,
	"link":  true,
	"img":   true,
	"embed": true,
	"param": true,
	"area":  true,
	"col":   true,
	"input": true,
}

// ----------------------------------------------------------------------------
// Types of issues to ignore

type htmlIssue uint32

const (
	issueIgnoreNone htmlIssue = 0
	issueDoctype    htmlIssue = 1 << (iota - 1)
	issueStructure
	issueUniqIDs
	issueLangTag
	issueAttr
	issueEscaping
	issueLabelRef
	issueURL
)

func ignoreMask(s string) (htmlIssue, error) {
	// what an ugly hack
	const issueNames = "doctype  structureuniqueidslang     attr     escaping label    url      "
	mask := htmlIssue(0)
	s = strings.ToLower(s)
	for _, p := range strings.Split(s, " ") {
		if p == "" {
			continue
		}
		i := strings.Index(issueNames, p)
		if i == -1 {
			return mask, fmt.Errorf("no such html issue '%s'", p)
		}
		mask |= 1 << uint(i/9)
	}
	return mask, nil
}

// ----------------------------------------------------------------------------
// htmlState

// htmlState collects information about a HTML document.
type htmlState struct {
	body string
	i    int
	line int

	elementCount map[string]int
	seenIDs      map[string]bool
	openTags     []string
	labelFor     []string
	errors       ErrorList

	ignore     htmlIssue
	badNesting bool
}

func newHtmlState(body string, ignore htmlIssue) *htmlState {
	return &htmlState{
		body:         body,
		i:            0,
		line:         0,
		elementCount: make(map[string]int, 50),
		seenIDs:      make(map[string]bool),
		openTags:     make([]string, 0, 50),
		labelFor:     make([]string, 0, 10),
		errors:       make(ErrorList, 0),
		badNesting:   false,
		ignore:       ignore,
	}
}

func (s *htmlState) Read(buf []byte) (int, error) {
	n := 0
	last := byte(0)
	for n < len(buf) && s.i < len(s.body) && last != '\n' {
		buf[n] = s.body[s.i]
		last = s.body[s.i]
		n++
		s.i++
	}
	s.line++
	if s.i == len(s.body) {
		return n, io.EOF
	}
	return n, nil
}

// err records the error e.
func (s *htmlState) err(e error) {
	s.errors = append(s.errors, PosError{Err: e, Line: s.line})
}

// count the tag
func (s *htmlState) count(tag string) {
	s.elementCount[tag] = s.elementCount[tag] + 1
}

// checkID chesk for duplicate ids.
func (s *htmlState) checkID(id string) {
	if s.ignore&issueUniqIDs == 0 {
		if _, seen := s.seenIDs[id]; seen {
			s.err(fmt.Errorf("duplicate id '%s'", id))
		}
	}
	s.seenIDs[id] = true
}

// record the id from a  tag.
func (s *htmlState) recordLabel(id string) {
	s.labelFor = append(s.labelFor, id)
}

// checkEscaping of text
func (s *htmlState) checkEscaping(text string) {
	if s.ignore&issueEscaping != 0 {
		return
	}

	// Javascript is full of unescaped <, > and && and content of 'noscript'
	// and 'iframe' seems to be unparsed by package html: Skip check of
	// proper escaping inside these elements.
	if n := len(s.openTags); n > 0 &&
		(s.openTags[n-1] == "script" || s.openTags[n-1] == "noscript" || s.openTags[n-1] == "iframe") {
		return
	}

	if strings.Index(text, "<") != -1 {
		s.err(fmt.Errorf("unescaped '<'"))
	}
	if strings.Index(text, ">") != -1 {
		s.err(fmt.Errorf("unescaped '>'"))
	}
	for len(text) > 0 {
		if i := strings.Index(text, "&"); i != -1 {
			text = text[i:]
			if strings.HasPrefix(text, "&") {
				text = text[5:]
			} else {
				ue := html.UnescapeString(text)
				if strings.HasPrefix(ue, "&") {
					s.err(fmt.Errorf("unescaped '&' or unknow entity"))
					break
				}
				text = text[1:]
			}
		} else {
			break
		}
	}
}

// checkLang tries to parse the language tag lang.
func (s *htmlState) checkLang(lang string) {
	if s.ignore&issueLangTag != 0 {
		return
	}
	_, err := language.Parse(lang)
	switch e := err.(type) {
	case nil:
		// No error.
	case language.ValueError:
		s.err(fmt.Errorf("language tag '%s' has bad part %s", lang, e.Subtag()))
	default:
		// A syntax error.
		s.err(fmt.Errorf("language tag '%s' is ill-formed", lang))
	}
}

// checkURL raw to be properly encoded. HTML escaping has been checked already.
// for now just dissalow spaces
func (s *htmlState) checkURL(raw string) {
	if s.ignore&issueURL != 0 {
		return
	}

	if strings.HasPrefix(raw, "mailto:") {
		if strings.Index(raw, "@") == -1 {
			s.err(fmt.Errorf("not an email address"))
		}
		return
	}

	u, err := url.Parse(raw)
	if err != nil {
		s.err(fmt.Errorf("bad URL '%s': %s", raw, err.Error()))
		return
	}
	if u.Opaque != "" {
		s.err(fmt.Errorf("bad URL part '%s'", u.Opaque))
		return
	}

	if strings.Index(raw, " ") != -1 {
		s.err(fmt.Errorf("unencoded space in URL"))
	}
}

// push tag on stack of open tags
func (s *htmlState) push(tag string) {
	s.openTags = append(s.openTags, tag)
}

// try to pop tag from stack of open tags, record error if failed.
func (s *htmlState) pop(tag string) {
	n := len(s.openTags)
	if n == 0 {
		if s.ignore&issueStructure == 0 {
			s.err(fmt.Errorf("no open tags left to close %s", tag))
		}
		return
	}
	pop := s.openTags[n-1]
	s.openTags = s.openTags[:n-1]
	if s.ignore&issueStructure != 0 {
		return
	}
	if pop != tag && !s.badNesting { // report broken structure just once.
		s.err(fmt.Errorf("tag '%s' closed by '%s'", pop, tag))
		s.badNesting = true
	}
}
<commit_msg>ht: document shortcommings of ValidHTML<commit_after>// Copyright 2015 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// validhtml.go contains checks to slighty validate a HTML document.

package ht

import (
	"fmt"
	"io"
	"net/url"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/text/language"
)

func init() {
	RegisterCheck(ValidHTML{})
}

// ValidHTML checks for valid HTML 5; well kinda: It make sure that some
// common but easy to detect fuckups are not present. The following issues
// are detected:
//   * 'doctype':   not exactly one DOCTYPE
//   * 'structure': ill-formed tag nesting / tag closing
//   * 'uniqueids': uniqness of id attribute values
//   * 'lang':      ill-formed lang attributes
//   * 'attr':      dupplicate attributes
//   * 'escaping':  unescaped &, < and > characters or unknown entities
//   * 'label':     reference to nonexisting id in label tags
//   * 'url':       malformed URLs
//
// Notes:
//  - HTML5 allows unescaped & in several circumstances but ValidHTML
//    reports all stray & as an error.
//  - The lang attributes are parse very lax, e.g. the non-canonical form
//    'de_CH' is considered valid (and equivalent to 'de-CH'). I don't
//    know how browser handle this.
type ValidHTML struct {
	// Ignore is a space seperated list of issues to ignore.
	// You normaly won't skip detection of these issues as all issues
	// are fundamental flaw which are easy to fix.
	Ignore string `json:",omitempty"`
}

// Execute implements Check's Execute method.
func (v ValidHTML) Execute(t *Test) error {
	if t.Response.BodyErr != nil {
		return ErrBadBody
	}

	mask, _ := ignoreMask(v.Ignore)
	state := newHtmlState(t.Response.BodyStr, mask)

	// Parse document and record local errors in state.
	z := html.NewTokenizer(state)
	depth := 0
done:
	for {
		tt := z.Next()
		// fmt.Printf("%s%s: ", strings.Repeat("  ", depth), tt)
		switch tt {
		case html.ErrorToken:
			if z.Err() == io.EOF {
				break done
			}
			return z.Err()
		case html.TextToken:
			// fmt.Printf(" %q\n", z.Text())
			// fmt.Println()
			if depth > 0 {
				state.checkEscaping(string(z.Raw()))
			}
		case html.StartTagToken, html.SelfClosingTagToken:
			raw := string(z.Raw())
			if len(raw) > 3 {
				state.checkEscaping(raw[1 : len(raw)-1])
			}
			tn, hasAttr := z.TagName()
			// Some tags are empty and may be written in the self-closing
			// variant "
" or simply empty like "
".
			// TODO: Maybe allow the non-compliant form "

" too?
			if tt != html.SelfClosingTagToken && !emptyHTMLElement[string(tn)] {
				state.push(string(tn))
				depth++
			}
			tag := string(tn)
			// fmt.Printf(" %s  ", tag)
			state.count(tag)
			attrs := map[string]string{}
			var bkey, bval []byte
			for hasAttr {
				bkey, bval, hasAttr = z.TagAttr()
				key, val := string(bkey), string(bval)
				// fmt.Printf("%s=%s ", key, val)
				if _, ok := attrs[key]; ok {
					if state.ignore&issueAttr == 0 {
						state.err(fmt.Errorf("duplicate attribute '%s'", key))
					}
				}
				attrs[key] = val
				switch {
				case key == "id":
					state.checkID(val)
				case tag == "label" && key == "for":
					state.recordLabel(val)
				case key == "lang":
					state.checkLang(val)
				case (tag == "a" && key == "href") ||
					(tag == "img" && key == "src"): // TODO link?
					state.checkURL(val)
				}
			}
			// fmt.Println()
		case html.EndTagToken:
			tn, _ := z.TagName()
			// fmt.Println(" ", string(tn))
			state.pop(string(tn))
			depth--
		case html.CommentToken:
		case html.DoctypeToken:
			state.count("DOCTYPE")
		}
	}

	// Check for global errors.
	state.line++ // Global errors are reported "after the last line".
	if state.ignore&issueDoctype == 0 {
		if d := state.elementCount["DOCTYPE"]; d != 1 {
			state.err(fmt.Errorf("found %d DOCTYPE", d))
		}
	}
	if state.ignore&issueLabelRef == 0 {
		for _, id := range state.labelFor {
			if _, ok := state.seenIDs[id]; !ok {
				state.err(fmt.Errorf("label references unknown id '%s'", id))
			}
		}
	}

	if len(state.errors) == 0 {
		return nil
	}

	if len(state.errors) == 0 {
		return nil
	}
	return state.errors
}

// Prepare implements Check's Prepare method.
func (v ValidHTML) Prepare() error {
	_, err := ignoreMask(v.Ignore)
	return err
}

// emptyHTMLElement is the list of all HTML5 elements which are empty, that
// is they are implecitely self-closing and can be written either in
// XML-style like e.g. "
" or in HTML5-style just "
".
// The list was taken from:
//   - http://www.elharo.com/blog/software-development/web-development/2007/01/29/all-empty-tags-in-html/
// Someone should check the whole list here: http://www.w3schools.com/tags/default.asp
var emptyHTMLElement = map[string]bool{
	"br":    true,
	"hr":    true,
	"meta":  true,
	"base":  true,
	"link":  true,
	"img":   true,
	"embed": true,
	"param": true,
	"area":  true,
	"col":   true,
	"input": true,
}

// ----------------------------------------------------------------------------
// Types of issues to ignore

type htmlIssue uint32

const (
	issueIgnoreNone htmlIssue = 0
	issueDoctype    htmlIssue = 1 << (iota - 1)
	issueStructure
	issueUniqIDs
	issueLangTag
	issueAttr
	issueEscaping
	issueLabelRef
	issueURL
)

func ignoreMask(s string) (htmlIssue, error) {
	// what an ugly hack
	const issueNames = "doctype  structureuniqueidslang     attr     escaping label    url      "
	mask := htmlIssue(0)
	s = strings.ToLower(s)
	for _, p := range strings.Split(s, " ") {
		if p == "" {
			continue
		}
		i := strings.Index(issueNames, p)
		if i == -1 {
			return mask, fmt.Errorf("no such html issue '%s'", p)
		}
		mask |= 1 << uint(i/9)
	}
	return mask, nil
}

// ----------------------------------------------------------------------------
// htmlState

// htmlState collects information about a HTML document.
type htmlState struct {
	body string
	i    int
	line int

	elementCount map[string]int
	seenIDs      map[string]bool
	openTags     []string
	labelFor     []string
	errors       ErrorList

	ignore     htmlIssue
	badNesting bool
}

func newHtmlState(body string, ignore htmlIssue) *htmlState {
	return &htmlState{
		body:         body,
		i:            0,
		line:         0,
		elementCount: make(map[string]int, 50),
		seenIDs:      make(map[string]bool),
		openTags:     make([]string, 0, 50),
		labelFor:     make([]string, 0, 10),
		errors:       make(ErrorList, 0),
		badNesting:   false,
		ignore:       ignore,
	}
}

func (s *htmlState) Read(buf []byte) (int, error) {
	n := 0
	last := byte(0)
	for n < len(buf) && s.i < len(s.body) && last != '\n' {
		buf[n] = s.body[s.i]
		last = s.body[s.i]
		n++
		s.i++
	}
	s.line++
	if s.i == len(s.body) {
		return n, io.EOF
	}
	return n, nil
}

// err records the error e.
func (s *htmlState) err(e error) {
	s.errors = append(s.errors, PosError{Err: e, Line: s.line})
}

// count the tag
func (s *htmlState) count(tag string) {
	s.elementCount[tag] = s.elementCount[tag] + 1
}

// checkID chesk for duplicate ids.
func (s *htmlState) checkID(id string) {
	if s.ignore&issueUniqIDs == 0 {
		if _, seen := s.seenIDs[id]; seen {
			s.err(fmt.Errorf("duplicate id '%s'", id))
		}
	}
	s.seenIDs[id] = true
}

// record the id from a  tag.
func (s *htmlState) recordLabel(id string) {
	s.labelFor = append(s.labelFor, id)
}

// checkEscaping of text
func (s *htmlState) checkEscaping(text string) {
	if s.ignore&issueEscaping != 0 {
		return
	}

	// Javascript is full of unescaped <, > and && and content of 'noscript'
	// and 'iframe' seems to be unparsed by package html: Skip check of
	// proper escaping inside these elements.
	if n := len(s.openTags); n > 0 &&
		(s.openTags[n-1] == "script" || s.openTags[n-1] == "noscript" || s.openTags[n-1] == "iframe") {
		return
	}

	if strings.Index(text, "<") != -1 {
		s.err(fmt.Errorf("unescaped '<'"))
	}
	if strings.Index(text, ">") != -1 {
		s.err(fmt.Errorf("unescaped '>'"))
	}
	for len(text) > 0 {
		if i := strings.Index(text, "&"); i != -1 {
			text = text[i:]
			if strings.HasPrefix(text, "&") {
				text = text[5:]
			} else {
				ue := html.UnescapeString(text)
				if strings.HasPrefix(ue, "&") {
					s.err(fmt.Errorf("unescaped '&' or unknow entity"))
					break
				}
				text = text[1:]
			}
		} else {
			break
		}
	}
}

// checkLang tries to parse the language tag lang.
func (s *htmlState) checkLang(lang string) {
	if s.ignore&issueLangTag != 0 {
		return
	}
	_, err := language.Parse(lang)
	switch e := err.(type) {
	case nil:
		// No error.
	case language.ValueError:
		s.err(fmt.Errorf("language tag '%s' has bad part %s", lang, e.Subtag()))
	default:
		// A syntax error.
		s.err(fmt.Errorf("language tag '%s' is ill-formed", lang))
	}
}

// checkURL raw to be properly encoded. HTML escaping has been checked already.
// for now just dissalow spaces
func (s *htmlState) checkURL(raw string) {
	if s.ignore&issueURL != 0 {
		return
	}

	if strings.HasPrefix(raw, "mailto:") {
		if strings.Index(raw, "@") == -1 {
			s.err(fmt.Errorf("not an email address"))
		}
		return
	}

	u, err := url.Parse(raw)
	if err != nil {
		s.err(fmt.Errorf("bad URL '%s': %s", raw, err.Error()))
		return
	}
	if u.Opaque != "" {
		s.err(fmt.Errorf("bad URL part '%s'", u.Opaque))
		return
	}

	if strings.Index(raw, " ") != -1 {
		s.err(fmt.Errorf("unencoded space in URL"))
	}
}

// push tag on stack of open tags
func (s *htmlState) push(tag string) {
	s.openTags = append(s.openTags, tag)
}

// try to pop tag from stack of open tags, record error if failed.
func (s *htmlState) pop(tag string) {
	n := len(s.openTags)
	if n == 0 {
		if s.ignore&issueStructure == 0 {
			s.err(fmt.Errorf("no open tags left to close %s", tag))
		}
		return
	}
	pop := s.openTags[n-1]
	s.openTags = s.openTags[:n-1]
	if s.ignore&issueStructure != 0 {
		return
	}
	if pop != tag && !s.badNesting { // report broken structure just once.
		s.err(fmt.Errorf("tag '%s' closed by '%s'", pop, tag))
		s.badNesting = true
	}
}
<|endoftext|>


<commit_before>package etcd

import (
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"

	e "github.com/coreos/etcd/client"
	"github.com/portworx/kvdb"
)

const (
	Name                          = "etcd-kv"
	defHost                       = "127.0.0.1:4001"
	defaultRetryCount             = 60
	defaultIntervalBetweenRetries = time.Millisecond * 500
)

type EtcdKV struct {
	client e.KeysAPI
	domain string
}

type etcdLock struct {
	done     chan struct{}
	unlocked bool
	err      error
	sync.Mutex
}

func EtcdInit(domain string,
	machines []string,
	options map[string]string,
) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	}
	cfg := e.Config{
		Endpoints: machines,
		Transport: e.DefaultTransport,
		// The time required for a request to fail - 10 sec
		HeaderTimeoutPerRequest: time.Duration(10) * time.Second,
	}
	c, err := e.New(cfg)
	if err != nil {
		return nil, err
	}
	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}
	kvapi := e.NewKeysAPI(c)
	kv := &EtcdKV{
		client: kvapi,
		domain: domain,
	}
	return kv, nil
}

func (kv *EtcdKV) String() string {
	return Name
}

func (kv *EtcdKV) nodeToKv(node *e.Node) *kvdb.KVPair {

	kvp := &kvdb.KVPair{
		Value:         []byte(node.Value),
		TTL:           node.TTL,
		ModifiedIndex: node.ModifiedIndex,
		CreatedIndex:  node.CreatedIndex,
	}

	// Strip out the leading '/'
	if len(node.Key) != 0 {
		kvp.Key = node.Key[1:]
	} else {
		kvp.Key = node.Key
	}
	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *EtcdKV) resultToKv(result *e.Response) *kvdb.KVPair {

	kvp := kv.nodeToKv(result.Node)
	switch result.Action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}
	kvp.KVDBIndex = result.Index
	return kvp
}

func (kv *EtcdKV) resultToKvs(result *e.Response) kvdb.KVPairs {

	kvs := make([]*kvdb.KVPair, len(result.Node.Nodes))
	for i := range result.Node.Nodes {
		kvs[i] = kv.nodeToKv(result.Node.Nodes[i])
		kvs[i].KVDBIndex = result.Index
	}
	return kvs
}

func (kv *EtcdKV) get(key string, recursive, sort bool) (*kvdb.KVPair, error) {
	var err error
	for i := 0; i < defaultRetryCount; i++ {
		result, err := kv.client.Get(context.Background(), key, &e.GetOptions{
			Recursive: recursive,
			Sort:      sort,
		})
		if err == nil {
			return kv.resultToKv(result), nil
		}

		switch err.(type) {
		case *e.ClusterError:
			fmt.Printf("kvdb get error: %v, retry count: %v\n", err, i)
			time.Sleep(defaultIntervalBetweenRetries)
		default:
			return nil, err
		}
	}
	return nil, err
}

func (kv *EtcdKV) Get(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key
	return kv.get(key, false, false)
}

func (kv *EtcdKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *EtcdKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *EtcdKV) setWithRetry(ctx context.Context, key, value string,
	opts *e.SetOptions) (*kvdb.KVPair, error) {
	var err error
	for i := 0; i < defaultRetryCount; i++ {
		result, err := kv.client.Set(ctx, key, value, opts)
		if err == nil {
			return kv.resultToKv(result), nil
		}
		switch err.(type) {
		case *e.ClusterError:
			fmt.Printf("kvdb set error: %v, retry count: %v\n", err, i)
			time.Sleep(defaultIntervalBetweenRetries)
		default:
			return nil, err
		}
	}
	return nil, err
}

func (kv *EtcdKV) Put(
	key string,
	val interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	key = kv.domain + key
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	return kv.setWithRetry(
		context.Background(),
		key,
		string(b),
		&e.SetOptions{
			TTL: time.Duration(ttl) * time.Second,
		})
}

func (kv *EtcdKV) Create(
	key string,
	val interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	return kv.setWithRetry(context.Background(), key, string(b), &e.SetOptions{
		TTL:       time.Duration(ttl) * time.Second,
		PrevExist: e.PrevNoExist,
	})
}

func (kv *EtcdKV) Update(
	key string,
	val interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	return kv.setWithRetry(context.Background(), key, string(b), &e.SetOptions{
		TTL:       time.Duration(ttl) * time.Second,
		PrevExist: e.PrevExist,
	})
}

func (kv *EtcdKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix
	var err error
	for i := 0; i < defaultRetryCount; i++ {
		result, err := kv.client.Get(context.Background(), prefix, &e.GetOptions{
			Recursive: true,
			Sort:      true,
		})
		if err == nil {
			return kv.resultToKvs(result), nil
		}
		switch err.(type) {
		case *e.ClusterError:
			fmt.Printf("kvdb set error: %v, retry count: %v\n", err, i)
			time.Sleep(defaultIntervalBetweenRetries)
		default:
			return nil, err
		}
	}
	return nil, err
}

func (kv *EtcdKV) Delete(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key

	result, err := kv.client.Delete(context.Background(), key, &e.DeleteOptions{
		Recursive: false,
	})
	if err == nil {
		return kv.resultToKv(result), err
	}
	return nil, err
}

func (kv *EtcdKV) DeleteTree(prefix string) error {
	prefix = kv.domain + prefix

	_, err := kv.client.Delete(context.Background(), prefix, &e.DeleteOptions{
		Recursive: true,
	})
	return err
}

func (kv *EtcdKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *EtcdKV) CompareAndSet(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
	prevValue []byte,
) (*kvdb.KVPair, error) {

	opts := &e.SetOptions{
		PrevValue: string(prevValue),
	}
	if (flags & kvdb.KVModifiedIndex) != 0 {
		opts.PrevIndex = kvp.ModifiedIndex
	}
	if (flags & kvdb.KVTTL) != 0 {
		opts.TTL = time.Duration(kvp.TTL)
	}
	result, err := kv.client.Set(
		context.Background(),
		kv.domain+kvp.Key,
		string(kvp.Value),
		opts,
	)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) CompareAndDelete(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
) (*kvdb.KVPair, error) {

	result, err := kv.client.Delete(
		context.Background(),
		kv.domain+kvp.Key,
		&e.DeleteOptions{
			PrevValue: string(kvp.Value),
			PrevIndex: kvp.ModifiedIndex,
		})
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) WatchKey(
	key string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
) error {

	key = kv.domain + key
	go kv.watchStart(key, false, waitIndex, opaque, cb)
	return nil
}

func (kv *EtcdKV) WatchTree(
	prefix string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
) error {

	prefix = kv.domain + prefix
	go kv.watchStart(prefix, true, waitIndex, opaque, cb)
	return nil
}

func (kv *EtcdKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key
	duration := time.Second

	count := 0
	kvPair, err := kv.Create(key, "locked", ttl)
	for maxCount := 300; err != nil && count < maxCount; count++ {
		time.Sleep(duration)
		kvPair, err = kv.Create(key, "locked", ttl)
	}
	if err != nil {
		return nil, err
	}
	if count > 10 {
		fmt.Printf("ETCD: spent %v iterations locking %v\n", count, key)
	}
	kvPair.TTL = int64(time.Duration(ttl) * time.Second)
	kvPair.Lock = &etcdLock{done: make(chan struct{})}
	go kv.refreshLock(kvPair)

	return kvPair, err
}

func (kv *EtcdKV) refreshLock(kvPair *kvdb.KVPair) {

	l := kvPair.Lock.(*etcdLock)
	ttl := kvPair.TTL
	refresh := time.NewTicker(time.Duration(kvPair.TTL) / 4)
	var keyString string
	if kvPair != nil {
		keyString = kvPair.Key
	}
	defer refresh.Stop()
	for {
		select {
		case <-refresh.C:
			l.Lock()
			for !l.unlocked {
				kvPair.TTL = ttl
				kvp, err := kv.CompareAndSet(
					kvPair,
					kvdb.KVTTL|kvdb.KVModifiedIndex,
					kvPair.Value,
				)
				if err != nil {
					fmt.Printf(
						"Error refreshing lock for key %v: %v\n",
						keyString, err,
					)
					l.err = err
					l.Unlock()
					return
				}
				kvPair.ModifiedIndex = kvp.ModifiedIndex
				break
			}
			l.Unlock()
		case <-l.done:
			return
		}
	}
}

func (kv *EtcdKV) Unlock(kvp *kvdb.KVPair) error {
	l, ok := kvp.Lock.(*etcdLock)
	if !ok {
		return fmt.Errorf("Invalid lock structure for key %v", string(kvp.Key))
	}
	l.Lock()
	// Don't modify kvp here, CompareAndDelete does that.
	_, err := kv.CompareAndDelete(kvp, kvdb.KVFlags(0))
	if err == nil {
		l.unlocked = true
		l.Unlock()
		l.done <- struct{}{}
		return nil
	}
	l.Unlock()
	return err
}

func (kv *EtcdKV) watchStart(key string,
	recursive bool,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
) {
	ctx, cancel := context.WithCancel(context.Background())
	watcher := kv.client.Watcher(key, &e.WatcherOptions{
		AfterIndex: waitIndex,
		Recursive:  recursive,
	})
	isCancelSent := false
	for {
		r, watchErr := watcher.Next(ctx)
		if watchErr != nil && !isCancelSent {
			e, ok := watchErr.(e.Error)
			if ok {
				fmt.Printf("Etcd error code %d Message %s Cause %s Index %ju\n",
					e.Code, e.Message, e.Cause, e.Index)
			} else {
				fmt.Printf("Etcd returned an error : %s", watchErr.Error())
			}
			cb(key, opaque, nil, watchErr)
		} else if watchErr == nil && !isCancelSent {
			err := cb(key, opaque, kv.resultToKv(r), nil)
			if err != nil {
				// Cancel the context
				cancel()
				isCancelSent = true
			}
		} else {
			cb(key, opaque, nil, kvdb.ErrWatchStopped)
			break
		}
	}
}

func (kv *EtcdKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func init() {
	kvdb.Register(Name, EtcdInit)
}
<commit_msg>get after update to handle intermediate failures<commit_after>package etcd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"

	e "github.com/coreos/etcd/client"
	"github.com/portworx/kvdb"
)

const (
	Name                          = "etcd-kv"
	defHost                       = "http://127.0.0.1:4001"
	defaultRetryCount             = 60
	defaultIntervalBetweenRetries = time.Millisecond * 500
)

type EtcdKV struct {
	client e.KeysAPI
	domain string
}

type etcdLock struct {
	done     chan struct{}
	unlocked bool
	err      error
	sync.Mutex
}

func EtcdInit(domain string,
	machines []string,
	options map[string]string,
) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	}
	cfg := e.Config{
		Endpoints: machines,
		Transport: e.DefaultTransport,
		// The time required for a request to fail - 30 sec
		HeaderTimeoutPerRequest: time.Duration(10) * time.Second,
	}
	c, err := e.New(cfg)
	if err != nil {
		return nil, err
	}
	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}
	kvapi := e.NewKeysAPI(c)
	kv := &EtcdKV{
		client: kvapi,
		domain: domain,
	}
	return kv, nil
}

func (kv *EtcdKV) String() string {
	return Name
}

func (kv *EtcdKV) nodeToKv(node *e.Node) *kvdb.KVPair {

	kvp := &kvdb.KVPair{
		Value:         []byte(node.Value),
		TTL:           node.TTL,
		ModifiedIndex: node.ModifiedIndex,
		CreatedIndex:  node.CreatedIndex,
	}

	// Strip out the leading '/'
	if len(node.Key) != 0 {
		kvp.Key = node.Key[1:]
	} else {
		kvp.Key = node.Key
	}
	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *EtcdKV) resultToKv(result *e.Response) *kvdb.KVPair {

	kvp := kv.nodeToKv(result.Node)
	switch result.Action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}
	kvp.KVDBIndex = result.Index
	return kvp
}

func (kv *EtcdKV) resultToKvs(result *e.Response) kvdb.KVPairs {

	kvs := make([]*kvdb.KVPair, len(result.Node.Nodes))
	for i := range result.Node.Nodes {
		kvs[i] = kv.nodeToKv(result.Node.Nodes[i])
		kvs[i].KVDBIndex = result.Index
	}
	return kvs
}

func (kv *EtcdKV) get(key string, recursive, sort bool) (*kvdb.KVPair, error) {
	var err error
	for i := 0; i < defaultRetryCount; i++ {
		result, err := kv.client.Get(context.Background(), key, &e.GetOptions{
			Recursive: recursive,
			Sort:      sort,
		})
		if err == nil {
			return kv.resultToKv(result), nil
		}

		switch err.(type) {
		case *e.ClusterError:
			fmt.Printf("kvdb get error: %v, retry count: %v\n", err, i)
			time.Sleep(defaultIntervalBetweenRetries)
		default:
			return nil, err
		}
	}
	return nil, err
}

func (kv *EtcdKV) Get(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key
	return kv.get(key, false, false)
}

func (kv *EtcdKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *EtcdKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *EtcdKV) setWithRetry(ctx context.Context, key, value string,
	opts *e.SetOptions) (*kvdb.KVPair, error) {
	var (
		err    error
		i      int
		result *e.Response
	)
	for i = 0; i < defaultRetryCount; i++ {
		result, err = kv.client.Set(ctx, key, value, opts)
		if err == nil {
			return kv.resultToKv(result), nil
		}
		switch err.(type) {
		case *e.ClusterError:
			fmt.Printf("kvdb set error: %v, retry count: %v\n", err, i)
			time.Sleep(defaultIntervalBetweenRetries)
		default:
			goto out
		}
	}

out:
	// It's possible that update succeeded but the re-update failed.
	if i > 0 && i < defaultRetryCount && err != nil {
		kvp, err := kv.get(key, false, false)
		if err == nil && bytes.Equal(kvp.Value, []byte(value)) {
			if opts.PrevExist == e.PrevNoExist {
				kvp.Action = kvdb.KVCreate
			} else {
				kvp.Action = kvdb.KVSet
			}
			return kvp, nil
		}
	}

	return nil, err
}

func (kv *EtcdKV) Put(
	key string,
	val interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	key = kv.domain + key
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	return kv.setWithRetry(
		context.Background(),
		key,
		string(b),
		&e.SetOptions{
			TTL: time.Duration(ttl) * time.Second,
		})
}

func (kv *EtcdKV) Create(
	key string,
	val interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	return kv.setWithRetry(context.Background(), key, string(b), &e.SetOptions{
		TTL:       time.Duration(ttl) * time.Second,
		PrevExist: e.PrevNoExist,
	})
}

func (kv *EtcdKV) Update(
	key string,
	val interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	return kv.setWithRetry(context.Background(), key, string(b), &e.SetOptions{
		TTL:       time.Duration(ttl) * time.Second,
		PrevExist: e.PrevExist,
	})
}

func (kv *EtcdKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix
	var err error
	for i := 0; i < defaultRetryCount; i++ {
		result, err := kv.client.Get(context.Background(), prefix, &e.GetOptions{
			Recursive: true,
			Sort:      true,
		})
		if err == nil {
			return kv.resultToKvs(result), nil
		}
		switch err.(type) {
		case *e.ClusterError:
			fmt.Printf("kvdb set error: %v, retry count: %v\n", err, i)
			time.Sleep(defaultIntervalBetweenRetries)
		default:
			return nil, err
		}
	}
	return nil, err
}

func (kv *EtcdKV) Delete(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key

	result, err := kv.client.Delete(context.Background(), key, &e.DeleteOptions{
		Recursive: false,
	})
	if err == nil {
		return kv.resultToKv(result), err
	}
	return nil, err
}

func (kv *EtcdKV) DeleteTree(prefix string) error {
	prefix = kv.domain + prefix

	_, err := kv.client.Delete(context.Background(), prefix, &e.DeleteOptions{
		Recursive: true,
	})
	return err
}

func (kv *EtcdKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *EtcdKV) CompareAndSet(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
	prevValue []byte,
) (*kvdb.KVPair, error) {

	opts := &e.SetOptions{
		PrevValue: string(prevValue),
	}
	if (flags & kvdb.KVModifiedIndex) != 0 {
		opts.PrevIndex = kvp.ModifiedIndex
	}
	if (flags & kvdb.KVTTL) != 0 {
		opts.TTL = time.Duration(kvp.TTL)
	}
	result, err := kv.client.Set(
		context.Background(),
		kv.domain+kvp.Key,
		string(kvp.Value),
		opts,
	)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) CompareAndDelete(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
) (*kvdb.KVPair, error) {

	result, err := kv.client.Delete(
		context.Background(),
		kv.domain+kvp.Key,
		&e.DeleteOptions{
			PrevValue: string(kvp.Value),
			PrevIndex: kvp.ModifiedIndex,
		})
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) WatchKey(
	key string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
) error {

	key = kv.domain + key
	go kv.watchStart(key, false, waitIndex, opaque, cb)
	return nil
}

func (kv *EtcdKV) WatchTree(
	prefix string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
) error {

	prefix = kv.domain + prefix
	go kv.watchStart(prefix, true, waitIndex, opaque, cb)
	return nil
}

func (kv *EtcdKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key
	duration := time.Second

	count := 0
	kvPair, err := kv.Create(key, "locked", ttl)
	for maxCount := 300; err != nil && count < maxCount; count++ {
		time.Sleep(duration)
		kvPair, err = kv.Create(key, "locked", ttl)
	}
	if err != nil {
		return nil, err
	}
	if count > 10 {
		fmt.Printf("ETCD: spent %v iterations locking %v\n", count, key)
	}
	kvPair.TTL = int64(time.Duration(ttl) * time.Second)
	kvPair.Lock = &etcdLock{done: make(chan struct{})}
	go kv.refreshLock(kvPair)

	return kvPair, err
}

func (kv *EtcdKV) refreshLock(kvPair *kvdb.KVPair) {

	l := kvPair.Lock.(*etcdLock)
	ttl := kvPair.TTL
	refresh := time.NewTicker(time.Duration(kvPair.TTL) / 4)
	var keyString string
	if kvPair != nil {
		keyString = kvPair.Key
	}
	defer refresh.Stop()
	for {
		select {
		case <-refresh.C:
			l.Lock()
			for !l.unlocked {
				kvPair.TTL = ttl
				kvp, err := kv.CompareAndSet(
					kvPair,
					kvdb.KVTTL|kvdb.KVModifiedIndex,
					kvPair.Value,
				)
				if err != nil {
					fmt.Printf(
						"Error refreshing lock for key %v: %v\n",
						keyString, err,
					)
					l.err = err
					l.Unlock()
					return
				}
				kvPair.ModifiedIndex = kvp.ModifiedIndex
				break
			}
			l.Unlock()
		case <-l.done:
			return
		}
	}
}

func (kv *EtcdKV) Unlock(kvp *kvdb.KVPair) error {
	l, ok := kvp.Lock.(*etcdLock)
	if !ok {
		return fmt.Errorf("Invalid lock structure for key %v", string(kvp.Key))
	}
	l.Lock()
	// Don't modify kvp here, CompareAndDelete does that.
	_, err := kv.CompareAndDelete(kvp, kvdb.KVFlags(0))
	if err == nil {
		l.unlocked = true
		l.Unlock()
		l.done <- struct{}{}
		return nil
	}
	l.Unlock()
	return err
}

func (kv *EtcdKV) watchStart(key string,
	recursive bool,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
) {
	ctx, cancel := context.WithCancel(context.Background())
	watcher := kv.client.Watcher(key, &e.WatcherOptions{
		AfterIndex: waitIndex,
		Recursive:  recursive,
	})
	isCancelSent := false
	for {
		r, watchErr := watcher.Next(ctx)
		if watchErr != nil && !isCancelSent {
			e, ok := watchErr.(e.Error)
			if ok {
				fmt.Printf("Etcd error code %d Message %s Cause %s Index %ju\n",
					e.Code, e.Message, e.Cause, e.Index)
			} else {
				fmt.Printf("Etcd returned an error : %s", watchErr.Error())
			}
			cb(key, opaque, nil, watchErr)
		} else if watchErr == nil && !isCancelSent {
			err := cb(key, opaque, kv.resultToKv(r), nil)
			if err != nil {
				// Cancel the context
				cancel()
				isCancelSent = true
			}
		} else {
			cb(key, opaque, nil, kvdb.ErrWatchStopped)
			break
		}
	}
}

func (kv *EtcdKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func init() {
	kvdb.Register(Name, EtcdInit)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/zlib"
	"bytes"
	"os"
    "fmt"
    "strconv"
    "encoding/binary"
    "path/filepath"
    "baidu.com/etl"
    "logd/lib"
    "io"
    "sync"
    "time"
    "logd/tcp_pack"
    "logd/integrity"
    "logd/loglib"
)

type etlOutputer struct {

	buffer chan bytes.Buffer
    saveDir string
    dataDir string
    headerDir string
    icDir string
    etlDir string
    etlDoneDir string
    etlFailDir string
    writers map[string]*os.File    //存日志的fd
    headerWriters map[string]*os.File   //存header的fd
    config  map[string]string
    ic *integrity.IntegrityChecker
    wq *lib.WaitQuit
}

//工厂初始化函数
func EtlOutputerInit(buffer chan bytes.Buffer, config map[string]string) (e etlOutputer) {

	e.buffer = buffer
    saveDir, _ := config["save_dir"]
    e.saveDir = saveDir
    e.dataDir = filepath.Join(saveDir, "log_data")
    e.headerDir = filepath.Join(saveDir, "headers")
    e.icDir = filepath.Join(saveDir, "received")
    e.etlDir = filepath.Join(saveDir, "etl")
    e.etlDoneDir = filepath.Join(saveDir, "etl_done")
    e.etlFailDir = filepath.Join(saveDir, "etl_fail")
    e.ic = integrity.NewIntegrityChecker(e.icDir)

    os.MkdirAll(e.dataDir, 0775)
    os.MkdirAll(e.headerDir, 0775)
    os.MkdirAll(e.etlDir, 0775)
    os.MkdirAll(e.etlDoneDir, 0775)
    os.MkdirAll(e.etlFailDir, 0775)

    e.writers = make(map[string]*os.File)
    e.headerWriters = make(map[string]*os.File)
    e.config = config
    e.wq = lib.NewWaitQuit("etl outputer", -1)   //不限退出超时时间，以便etl能做完

	return e
}

func (e *etlOutputer) Start() {
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("etl outputer panic:%v", err))
        }

        e.wq.AllDone()
    }()

    spiderList, _ := e.config["spider_list"]
    colsFile , _ := e.config["columns_file"]

    if colsFile != "" {
        e.runEtl(spiderList, colsFile)
    }else{
        loglib.Error("[error] miss columns map file!")
    }
}

func (e *etlOutputer) Quit() bool {
    return e.wq.Quit()
}

func (e *etlOutputer) runEtl(spiderList string, colsFile string) {
    wg := &sync.WaitGroup{}
    fkeyChan := make(chan string, 100)
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("runEtl() panic:%v", err))
        }

        e.ic.SaveStatus()
        close(fkeyChan)
        //等待etl routine结束
        wg.Wait()
    }()

    for i:=0; i<2; i++ {
        wg.Add(1)
        go e.doEtl(fkeyChan, e.dataDir, e.etlDir, e.etlDoneDir, e.etlFailDir, spiderList, colsFile, wg)
    }
    nextCheckTime := time.Now().Add(10 * time.Minute)
    //使用range遍历，方便安全退出，只要发送方退出时关闭chan，这里就可以退出了
    for b := range e.buffer {
        loglib.Info(fmt.Sprintf("pack in chan: %d", len(e.buffer)))
        buf := make([]byte, 4)
        bp := &b
        bp.Read(buf)

        l, _ := binary.Uvarint(buf)
        headerLen := int(l)
        //get pack header
        buf = make([]byte, headerLen)
        bp.Read(buf)
        header := tcp_pack.ParseHeader(buf)

        r, err := zlib.NewReader(bp)
        if err != nil {
            loglib.Error("zlib reader Error: " + err.Error())
        }else{
            lines, _ := strconv.Atoi(header["lines"])
            done := false
            if header["done"] == "1" {
                done = true
            }
            e.ic.Add(header["ip"], header["hour"], header["id"], lines, done)

            writerKey := header["ip"] + "_" + header["hour"]
            fout := e.getWriter(e.writers, e.dataDir, writerKey)

            buf = append(buf, '\n')
            /*
            //一头一尾写头信息，节省硬盘
            n, err := fout.Write(buf)
            if err != nil {
                loglib.Info(fmt.Sprintf("write %s %d %s", writerKey, n, err.Error()))
            }
            */
            nn, err := io.Copy(fout, r)
            if err != nil {
                loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
            }
            //fout.Write(buf)
            //单独存一份header便于查数
            fout = e.getWriter(e.headerWriters, e.headerDir, writerKey)
            n, err := fout.Write(buf)
            if err != nil {
                loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
            }
            //增加十分钟check一次的规则，避免done包先到，其他的包未到，则可能要等到下一小时才能check
            if done || time.Now().Unix() > nextCheckTime.Unix() {
                hourFinish, _ := e.ic.Check()
                for ip, hours := range hourFinish {
                    for _, hour := range hours {
                        writerKey = ip + "_" + hour
                        e.closeWriter(e.writers, writerKey)
                        e.closeWriter(e.headerWriters, writerKey)
                        loglib.Info(fmt.Sprintf("fkeychan %d", len(fkeyChan)))
                        fkeyChan <- writerKey
                    }
                }
                nextCheckTime = time.Now().Add(10 * time.Minute)
            }

            r.Close()
        }
    }
}

func (e *etlOutputer) getWriter(writers map[string]*os.File, parentDir string, key string) *os.File {
    w, ok := writers[key]
    if !ok || w == nil {
        fname := filepath.Join(parentDir, key)
        w1, err := os.OpenFile(fname, os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0666)
        writers[key] = w1
        w = w1
        if err != nil {
            loglib.Error(fmt.Sprintf("file outputer create writer: %s error: %s", fname, err.Error()))
        }
    }
    return w
}

func (e *etlOutputer) closeWriter(writers map[string]*os.File, key string) {
    w, ok := writers[key]
    if ok {
        if w != nil {
            w.Close()
        }
        delete(writers, key)
    }
}

func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, wg *sync.WaitGroup) {
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("doEtl() panic:%v", err))
        }

        wg.Done()
    }()
    loglib.Info("etl routine start")
    for fkey := range fkeyChan {
        d := etl.NewDispatcher(colsFile, etlDir, 5, fkey)
        g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d)
        go g.Start(false)

        fname := filepath.Join(logDataDir, fkey)
        loglib.Info("start etl for " + fname)

        err := g.ParseFile(fname)
        g.Wait()
        // etl success
        // mark success
        if err == nil {
            //采用循环，增加打tag的成功率
            for i:=0; i<5; i++ {
                fd, err := os.Create(filepath.Join(etlDoneDir, fkey))
                if err == nil {
                    fd.Close()
                    loglib.Info("finish etl for " + fname)
                    break
                }else{
                    loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error())
                }
            }
        }else{
            //采用循环，增加打tag的成功率
            for i:=0; i<5; i++ {
                fd, err := os.Create(filepath.Join(etlFailDir, fkey))
                if err == nil {
                    fd.Close()
                    loglib.Info("failed etl for " + fname)
                    break
                }else{
                    loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error())
                }
            }

        }
    }
    loglib.Info("etl routine finish")
}
<commit_msg>avoid writer be occupied too long<commit_after>package main

import (
	"compress/zlib"
	"bytes"
	"os"
    "fmt"
    "strconv"
    "encoding/binary"
    "path/filepath"
    "baidu.com/etl"
    "logd/lib"
    "io"
    "sync"
    "time"
    "logd/tcp_pack"
    "logd/integrity"
    "logd/loglib"
)

type etlOutputer struct {

	buffer chan bytes.Buffer
    saveDir string
    dataDir string
    headerDir string
    icDir string
    etlDir string
    etlDoneDir string
    etlFailDir string
    writers map[string]*os.File    //存日志的fd
    headerWriters map[string]*os.File   //存header的fd
    config  map[string]string
    ic *integrity.IntegrityChecker
    wq *lib.WaitQuit
}

//工厂初始化函数
func EtlOutputerInit(buffer chan bytes.Buffer, config map[string]string) (e etlOutputer) {

	e.buffer = buffer
    saveDir, _ := config["save_dir"]
    e.saveDir = saveDir
    e.dataDir = filepath.Join(saveDir, "log_data")
    e.headerDir = filepath.Join(saveDir, "headers")
    e.icDir = filepath.Join(saveDir, "received")
    e.etlDir = filepath.Join(saveDir, "etl")
    e.etlDoneDir = filepath.Join(saveDir, "etl_done")
    e.etlFailDir = filepath.Join(saveDir, "etl_fail")
    e.ic = integrity.NewIntegrityChecker(e.icDir)

    os.MkdirAll(e.dataDir, 0775)
    os.MkdirAll(e.headerDir, 0775)
    os.MkdirAll(e.etlDir, 0775)
    os.MkdirAll(e.etlDoneDir, 0775)
    os.MkdirAll(e.etlFailDir, 0775)

    e.writers = make(map[string]*os.File)
    e.headerWriters = make(map[string]*os.File)
    e.config = config
    e.wq = lib.NewWaitQuit("etl outputer", -1)   //不限退出超时时间，以便etl能做完

	return e
}

func (e *etlOutputer) Start() {
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("etl outputer panic:%v", err))
        }

        e.wq.AllDone()
    }()

    spiderList, _ := e.config["spider_list"]
    colsFile , _ := e.config["columns_file"]

    if colsFile != "" {
        e.runEtl(spiderList, colsFile)
    }else{
        loglib.Error("[error] miss columns map file!")
    }
}

func (e *etlOutputer) Quit() bool {
    return e.wq.Quit()
}

func (e *etlOutputer) runEtl(spiderList string, colsFile string) {
    wg := &sync.WaitGroup{}
    fkeyChan := make(chan string, 100)
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("runEtl() panic:%v", err))
        }

        e.ic.SaveStatus()
        e.closeWriters(e.writers)
        e.closeWriters(e.headerWriters)
        close(fkeyChan)
        //等待etl routine结束
        wg.Wait()
    }()

    for i:=0; i<2; i++ {
        wg.Add(1)
        go e.doEtl(fkeyChan, e.dataDir, e.etlDir, e.etlDoneDir, e.etlFailDir, spiderList, colsFile, wg)
    }
    nextCheckTime := time.Now().Add(10 * time.Minute)
    //使用range遍历，方便安全退出，只要发送方退出时关闭chan，这里就可以退出了
    for b := range e.buffer {
        loglib.Info(fmt.Sprintf("pack in chan: %d", len(e.buffer)))
        buf := make([]byte, 4)
        bp := &b
        bp.Read(buf)

        l, _ := binary.Uvarint(buf)
        headerLen := int(l)
        //get pack header
        buf = make([]byte, headerLen)
        bp.Read(buf)
        header := tcp_pack.ParseHeader(buf)

        r, err := zlib.NewReader(bp)
        if err != nil {
            loglib.Error("zlib reader Error: " + err.Error())
        }else{
            lines, _ := strconv.Atoi(header["lines"])
            done := false
            if header["done"] == "1" {
                done = true
            }
            e.ic.Add(header["ip"], header["hour"], header["id"], lines, done)

            writerKey := header["ip"] + "_" + header["hour"]
            fout := e.getWriter(e.writers, e.dataDir, writerKey)

            buf = append(buf, '\n')
            /*
            //一头一尾写头信息，节省硬盘
            n, err := fout.Write(buf)
            if err != nil {
                loglib.Info(fmt.Sprintf("write %s %d %s", writerKey, n, err.Error()))
            }
            */
            nn, err := io.Copy(fout, r)
            if err != nil {
                loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
            }
            //fout.Write(buf)
            //单独存一份header便于查数
            fout = e.getWriter(e.headerWriters, e.headerDir, writerKey)
            n, err := fout.Write(buf)
            if err != nil {
                loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
            }
            //增加十分钟check一次的规则，避免done包先到，其他的包未到，则可能要等到下一小时才能check
            if done || time.Now().Unix() > nextCheckTime.Unix() {
                hourFinish, _ := e.ic.Check()
                for ip, hours := range hourFinish {
                    for _, hour := range hours {
                        writerKey = ip + "_" + hour
                        loglib.Info(fmt.Sprintf("fkeychan %d", len(fkeyChan)))
                        fkeyChan <- writerKey
                    }
                }
                e.closeWriters(e.writers)
                e.closeWriters(e.headerWriters)
                nextCheckTime = time.Now().Add(10 * time.Minute)
            }

            r.Close()
        }
    }
}

func (e *etlOutputer) getWriter(writers map[string]*os.File, parentDir string, key string) *os.File {
    w, ok := writers[key]
    if !ok || w == nil {
        fname := filepath.Join(parentDir, key)
        w1, err := os.OpenFile(fname, os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0666)
        writers[key] = w1
        w = w1
        if err != nil {
            loglib.Error(fmt.Sprintf("file outputer create writer: %s error: %s", fname, err.Error()))
        }
    }
    return w
}

func (e *etlOutputer) closeWriter(writers map[string]*os.File, key string) {
    w, ok := writers[key]
    if ok {
        if w != nil {
            w.Close()
        }
        delete(writers, key)
    }
}
//关闭全部writer
func (e *etlOutputer) closeWriters(writers map[string]*os.File) {
    for key, w := range writers {
        if w != nil {
            w.Close()
        }
        delete(writers, key)
    }

}
func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, wg *sync.WaitGroup) {
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("doEtl() panic:%v", err))
        }

        wg.Done()
    }()
    loglib.Info("etl routine start")
    for fkey := range fkeyChan {
        d := etl.NewDispatcher(colsFile, etlDir, 5, fkey)
        g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d)
        go g.Start(false)

        fname := filepath.Join(logDataDir, fkey)
        loglib.Info("start etl for " + fname)

        err := g.ParseFile(fname)
        g.Wait()
        // etl success
        // mark success
        if err == nil {
            //采用循环，增加打tag的成功率
            for i:=0; i<5; i++ {
                fd, err := os.Create(filepath.Join(etlDoneDir, fkey))
                if err == nil {
                    fd.Close()
                    loglib.Info("finish etl for " + fname)
                    break
                }else{
                    loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error())
                }
            }
        }else{
            //采用循环，增加打tag的成功率
            for i:=0; i<5; i++ {
                fd, err := os.Create(filepath.Join(etlFailDir, fkey))
                if err == nil {
                    fd.Close()
                    loglib.Info("failed etl for " + fname)
                    break
                }else{
                    loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error())
                }
            }

        }
    }
    loglib.Info("etl routine finish")
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes_test

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"os"
	"sort"
)

func ExampleBuffer() {
	var b bytes.Buffer // A Buffer needs no initialization.
	b.Write([]byte("Hello "))
	fmt.Fprintf(&b, "world!")
	b.WriteTo(os.Stdout)
	// Output: Hello world!
}

func ExampleBuffer_reader() {
	// A Buffer can turn a string or a []byte into an io.Reader.
	buf := bytes.NewBufferString("R29waGVycyBydWxlIQ==")
	dec := base64.NewDecoder(base64.StdEncoding, buf)
	io.Copy(os.Stdout, dec)
	// Output: Gophers rule!
}

func ExampleCompare() {
	// Interpret Compare's result by comparing it to zero.
	var a, b []byte
	if bytes.Compare(a, b) < 0 {
		// a less b
	}
	if bytes.Compare(a, b) <= 0 {
		// a less or equal b
	}
	if bytes.Compare(a, b) > 0 {
		// a greater b
	}
	if bytes.Compare(a, b) >= 0 {
		// a greater or equal b
	}

	// Prefer Equal to Compare for equality comparisons.
	if bytes.Equal(a, b) {
		// a equal b
	}
	if !bytes.Equal(a, b) {
		// a not equal b
	}
}

func ExampleCompare_search() {
	// Binary search to find a matching byte slice.
	var needle []byte
	var haystack [][]byte // Assume sorted
	i := sort.Search(len(haystack), func(i int) bool {
		// Return needle <= haystack[i].
		return bytes.Compare(needle, haystack[i]) <= 0
	})
	if i < len(haystack) && bytes.Equal(needle, haystack[i]) {
		// Found it!
	}
}
<commit_msg>bytes: Change Compare example to be consistent with sort.Search's.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes_test

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"os"
	"sort"
)

func ExampleBuffer() {
	var b bytes.Buffer // A Buffer needs no initialization.
	b.Write([]byte("Hello "))
	fmt.Fprintf(&b, "world!")
	b.WriteTo(os.Stdout)
	// Output: Hello world!
}

func ExampleBuffer_reader() {
	// A Buffer can turn a string or a []byte into an io.Reader.
	buf := bytes.NewBufferString("R29waGVycyBydWxlIQ==")
	dec := base64.NewDecoder(base64.StdEncoding, buf)
	io.Copy(os.Stdout, dec)
	// Output: Gophers rule!
}

func ExampleCompare() {
	// Interpret Compare's result by comparing it to zero.
	var a, b []byte
	if bytes.Compare(a, b) < 0 {
		// a less b
	}
	if bytes.Compare(a, b) <= 0 {
		// a less or equal b
	}
	if bytes.Compare(a, b) > 0 {
		// a greater b
	}
	if bytes.Compare(a, b) >= 0 {
		// a greater or equal b
	}

	// Prefer Equal to Compare for equality comparisons.
	if bytes.Equal(a, b) {
		// a equal b
	}
	if !bytes.Equal(a, b) {
		// a not equal b
	}
}

func ExampleCompare_search() {
	// Binary search to find a matching byte slice.
	var needle []byte
	var haystack [][]byte // Assume sorted
	i := sort.Search(len(haystack), func(i int) bool {
		// Return haystack[i] >= needle.
		return bytes.Compare(haystack[i], needle) >= 0
	})
	if i < len(haystack) && bytes.Equal(haystack[i], needle) {
		// Found it!
	}
}
<|endoftext|>


<commit_before>package bytes_test

import (
	. "bytes"
	"encoding/base64"
	"io"
	"os"
)

// Hello world!
func ExampleBuffer() {
	var b Buffer // A Buffer needs no initialization.
	b.Write([]byte("Hello "))
	b.Write([]byte("world!"))
	b.WriteTo(os.Stdout)
}

// Gophers rule!
func ExampleBuffer_reader() {
	// A Buffer can turn a string or a []byte into an io.Reader.
	buf := NewBufferString("R29waGVycyBydWxlIQ==")
	dec := base64.NewDecoder(base64.StdEncoding, buf)
	io.Copy(os.Stdout, dec)
}
<commit_msg>pkg/bytes: add the usual copyright notice to example_test.go<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes_test

import (
	. "bytes"
	"encoding/base64"
	"io"
	"os"
)

// Hello world!
func ExampleBuffer() {
	var b Buffer // A Buffer needs no initialization.
	b.Write([]byte("Hello "))
	b.Write([]byte("world!"))
	b.WriteTo(os.Stdout)
}

// Gophers rule!
func ExampleBuffer_reader() {
	// A Buffer can turn a string or a []byte into an io.Reader.
	buf := NewBufferString("R29waGVycyBydWxlIQ==")
	dec := base64.NewDecoder(base64.StdEncoding, buf)
	io.Copy(os.Stdout, dec)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package token

import (
	"gob"
	"io"
	"os"
)

type serializedFile struct {
	// fields correspond 1:1 to fields with same (lower-case) name in File
	Name  string
	Base  int
	Size  int
	Lines []int
	Infos []lineInfo
}

type serializedFileSet struct {
	Base  int
	Files []serializedFile
}

// Read reads the fileset from r into s; s must not be nil.
func (s *FileSet) Read(r io.Reader) os.Error {
	var ss serializedFileSet
	if err := gob.NewDecoder(r).Decode(&ss); err != nil {
		return err
	}

	s.mutex.Lock()
	s.base = ss.Base
	files := make([]*File, len(ss.Files))
	for i := 0; i < len(ss.Files); i++ {
		f := &ss.Files[i]
		files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos}
	}
	s.files = files
	s.last = nil
	s.mutex.Unlock()

	return nil
}

// Write writes the fileset s to w.
func (s *FileSet) Write(w io.Writer) os.Error {
	var ss serializedFileSet

	s.mutex.Lock()
	ss.Base = s.base
	files := make([]serializedFile, len(s.files))
	for i, f := range s.files {
		files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos}
	}
	ss.Files = files
	s.mutex.Unlock()

	return gob.NewEncoder(w).Encode(ss)
}
<commit_msg>go/token: document deserialization property<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package token

import (
	"gob"
	"io"
	"os"
)

type serializedFile struct {
	// fields correspond 1:1 to fields with same (lower-case) name in File
	Name  string
	Base  int
	Size  int
	Lines []int
	Infos []lineInfo
}

type serializedFileSet struct {
	Base  int
	Files []serializedFile
}

func (s *serializedFileSet) Read(r io.Reader) os.Error {
	return gob.NewDecoder(r).Decode(s)
}

func (s *serializedFileSet) Write(w io.Writer) os.Error {
	return gob.NewEncoder(w).Encode(s)
}

// Read reads the fileset from r into s; s must not be nil.
// If r does not also implement io.ByteReader, it will be wrapped in a bufio.Reader.
func (s *FileSet) Read(r io.Reader) os.Error {
	var ss serializedFileSet
	if err := ss.Read(r); err != nil {
		return err
	}

	s.mutex.Lock()
	s.base = ss.Base
	files := make([]*File, len(ss.Files))
	for i := 0; i < len(ss.Files); i++ {
		f := &ss.Files[i]
		files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos}
	}
	s.files = files
	s.last = nil
	s.mutex.Unlock()

	return nil
}

// Write writes the fileset s to w.
func (s *FileSet) Write(w io.Writer) os.Error {
	var ss serializedFileSet

	s.mutex.Lock()
	ss.Base = s.base
	files := make([]serializedFile, len(s.files))
	for i, f := range s.files {
		files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos}
	}
	ss.Files = files
	s.mutex.Unlock()

	return ss.Write(w)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Parse "zoneinfo" time zone file.
// This is a fairly standard file format used on OS X, Linux, BSD, Sun, and others.
// See tzfile(5), http://en.wikipedia.org/wiki/Zoneinfo,
// and ftp://munnari.oz.au/pub/oldtz/

package time

import (
	"io/ioutil"
	"os"
	"sync"
)

const (
	headerSize = 4 + 16 + 4*7
	zoneDir    = "/usr/share/zoneinfo/"
	zoneDir2   = "/usr/share/lib/zoneinfo/"
)

// Simple I/O interface to binary blob of data.
type data struct {
	p     []byte
	error bool
}

func (d *data) read(n int) []byte {
	if len(d.p) < n {
		d.p = nil
		d.error = true
		return nil
	}
	p := d.p[0:n]
	d.p = d.p[n:]
	return p
}

func (d *data) big4() (n uint32, ok bool) {
	p := d.read(4)
	if len(p) < 4 {
		d.error = true
		return 0, false
	}
	return uint32(p[0])<<24 | uint32(p[1])<<16 | uint32(p[2])<<8 | uint32(p[3]), true
}

func (d *data) byte() (n byte, ok bool) {
	p := d.read(1)
	if len(p) < 1 {
		d.error = true
		return 0, false
	}
	return p[0], true
}

// Make a string by stopping at the first NUL
func byteString(p []byte) string {
	for i := 0; i < len(p); i++ {
		if p[i] == 0 {
			return string(p[0:i])
		}
	}
	return string(p)
}

// Parsed representation
type zone struct {
	utcoff int
	isdst  bool
	name   string
}

type zonetime struct {
	time         int32 // transition time, in seconds since 1970 GMT
	zone         *zone // the zone that goes into effect at that time
	isstd, isutc bool  // ignored - no idea what these mean
}

func parseinfo(bytes []byte) (zt []zonetime, ok bool) {
	d := data{bytes, false}

	// 4-byte magic "TZif"
	if magic := d.read(4); string(magic) != "TZif" {
		return nil, false
	}

	// 1-byte version, then 15 bytes of padding
	var p []byte
	if p = d.read(16); len(p) != 16 || p[0] != 0 && p[0] != '2' {
		return nil, false
	}

	// six big-endian 32-bit integers:
	//	number of UTC/local indicators
	//	number of standard/wall indicators
	//	number of leap seconds
	//	number of transition times
	//	number of local time zones
	//	number of characters of time zone abbrev strings
	const (
		NUTCLocal = iota
		NStdWall
		NLeap
		NTime
		NZone
		NChar
	)
	var n [6]int
	for i := 0; i < 6; i++ {
		nn, ok := d.big4()
		if !ok {
			return nil, false
		}
		n[i] = int(nn)
	}

	// Transition times.
	txtimes := data{d.read(n[NTime] * 4), false}

	// Time zone indices for transition times.
	txzones := d.read(n[NTime])

	// Zone info structures
	zonedata := data{d.read(n[NZone] * 6), false}

	// Time zone abbreviations.
	abbrev := d.read(n[NChar])

	// Leap-second time pairs
	d.read(n[NLeap] * 8)

	// Whether tx times associated with local time types
	// are specified as standard time or wall time.
	isstd := d.read(n[NStdWall])

	// Whether tx times associated with local time types
	// are specified as UTC or local time.
	isutc := d.read(n[NUTCLocal])

	if d.error { // ran out of data
		return nil, false
	}

	// If version == 2, the entire file repeats, this time using
	// 8-byte ints for txtimes and leap seconds.
	// We won't need those until 2106.

	// Now we can build up a useful data structure.
	// First the zone information.
	//	utcoff[4] isdst[1] nameindex[1]
	z := make([]zone, n[NZone])
	for i := 0; i < len(z); i++ {
		var ok bool
		var n uint32
		if n, ok = zonedata.big4(); !ok {
			return nil, false
		}
		z[i].utcoff = int(n)
		var b byte
		if b, ok = zonedata.byte(); !ok {
			return nil, false
		}
		z[i].isdst = b != 0
		if b, ok = zonedata.byte(); !ok || int(b) >= len(abbrev) {
			return nil, false
		}
		z[i].name = byteString(abbrev[b:])
	}

	// Now the transition time info.
	zt = make([]zonetime, n[NTime])
	for i := 0; i < len(zt); i++ {
		var ok bool
		var n uint32
		if n, ok = txtimes.big4(); !ok {
			return nil, false
		}
		zt[i].time = int32(n)
		if int(txzones[i]) >= len(z) {
			return nil, false
		}
		zt[i].zone = &z[txzones[i]]
		if i < len(isstd) {
			zt[i].isstd = isstd[i] != 0
		}
		if i < len(isutc) {
			zt[i].isutc = isutc[i] != 0
		}
	}
	return zt, true
}

func readinfofile(name string) ([]zonetime, bool) {
	buf, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, false
	}
	return parseinfo(buf)
}

var zones []zonetime
var onceSetupZone sync.Once

func setupZone() {
	// consult $TZ to find the time zone to use.
	// no $TZ means use the system default /etc/localtime.
	// $TZ="" means use UTC.
	// $TZ="foo" means use /usr/share/zoneinfo/foo.

	tz, err := os.Getenverror("TZ")
	switch {
	case err == os.ENOENV:
		zones, _ = readinfofile("/etc/localtime")
	case len(tz) > 0:
		var ok bool
		zones, ok = readinfofile(zoneDir + tz)
		if !ok {
			zones, _ = readinfofile(zoneDir2 + tz)
		}
	case len(tz) == 0:
		// do nothing: use UTC
	}
}

// Look up the correct time zone (daylight savings or not) for the given unix time, in the current location.
func lookupTimezone(sec int64) (zone string, offset int) {
	onceSetupZone.Do(setupZone)
	if len(zones) == 0 {
		return "UTC", 0
	}

	// Binary search for entry with largest time <= sec
	tz := zones
	for len(tz) > 1 {
		m := len(tz) / 2
		if sec < int64(tz[m].time) {
			tz = tz[0:m]
		} else {
			tz = tz[m:]
		}
	}
	z := tz[0].zone
	return z.name, z.utcoff
}

// lookupByName returns the time offset for the
// time zone with the given abbreviation. It only considers
// time zones that apply to the current system.
// For example, for a system configured as being in New York,
// it only recognizes "EST" and "EDT".
// For a system in San Francisco, "PST" and "PDT".
// For a system in Sydney, "EST" and "EDT", though they have
// different meanings than they do in New York.
func lookupByName(name string) (off int, found bool) {
	onceSetupZone.Do(setupZone)
	for _, z := range zones {
		if name == z.zone.name {
			return z.zone.utcoff, true
		}
	}
	return 0, false
}
<commit_msg>time: Support Irix 6 location for zoneinfo files.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Parse "zoneinfo" time zone file.
// This is a fairly standard file format used on OS X, Linux, BSD, Sun, and others.
// See tzfile(5), http://en.wikipedia.org/wiki/Zoneinfo,
// and ftp://munnari.oz.au/pub/oldtz/

package time

import (
	"io/ioutil"
	"os"
	"sync"
)

const (
	headerSize = 4 + 16 + 4*7
)

// Simple I/O interface to binary blob of data.
type data struct {
	p     []byte
	error bool
}

func (d *data) read(n int) []byte {
	if len(d.p) < n {
		d.p = nil
		d.error = true
		return nil
	}
	p := d.p[0:n]
	d.p = d.p[n:]
	return p
}

func (d *data) big4() (n uint32, ok bool) {
	p := d.read(4)
	if len(p) < 4 {
		d.error = true
		return 0, false
	}
	return uint32(p[0])<<24 | uint32(p[1])<<16 | uint32(p[2])<<8 | uint32(p[3]), true
}

func (d *data) byte() (n byte, ok bool) {
	p := d.read(1)
	if len(p) < 1 {
		d.error = true
		return 0, false
	}
	return p[0], true
}

// Make a string by stopping at the first NUL
func byteString(p []byte) string {
	for i := 0; i < len(p); i++ {
		if p[i] == 0 {
			return string(p[0:i])
		}
	}
	return string(p)
}

// Parsed representation
type zone struct {
	utcoff int
	isdst  bool
	name   string
}

type zonetime struct {
	time         int32 // transition time, in seconds since 1970 GMT
	zone         *zone // the zone that goes into effect at that time
	isstd, isutc bool  // ignored - no idea what these mean
}

func parseinfo(bytes []byte) (zt []zonetime, ok bool) {
	d := data{bytes, false}

	// 4-byte magic "TZif"
	if magic := d.read(4); string(magic) != "TZif" {
		return nil, false
	}

	// 1-byte version, then 15 bytes of padding
	var p []byte
	if p = d.read(16); len(p) != 16 || p[0] != 0 && p[0] != '2' {
		return nil, false
	}

	// six big-endian 32-bit integers:
	//	number of UTC/local indicators
	//	number of standard/wall indicators
	//	number of leap seconds
	//	number of transition times
	//	number of local time zones
	//	number of characters of time zone abbrev strings
	const (
		NUTCLocal = iota
		NStdWall
		NLeap
		NTime
		NZone
		NChar
	)
	var n [6]int
	for i := 0; i < 6; i++ {
		nn, ok := d.big4()
		if !ok {
			return nil, false
		}
		n[i] = int(nn)
	}

	// Transition times.
	txtimes := data{d.read(n[NTime] * 4), false}

	// Time zone indices for transition times.
	txzones := d.read(n[NTime])

	// Zone info structures
	zonedata := data{d.read(n[NZone] * 6), false}

	// Time zone abbreviations.
	abbrev := d.read(n[NChar])

	// Leap-second time pairs
	d.read(n[NLeap] * 8)

	// Whether tx times associated with local time types
	// are specified as standard time or wall time.
	isstd := d.read(n[NStdWall])

	// Whether tx times associated with local time types
	// are specified as UTC or local time.
	isutc := d.read(n[NUTCLocal])

	if d.error { // ran out of data
		return nil, false
	}

	// If version == 2, the entire file repeats, this time using
	// 8-byte ints for txtimes and leap seconds.
	// We won't need those until 2106.

	// Now we can build up a useful data structure.
	// First the zone information.
	//	utcoff[4] isdst[1] nameindex[1]
	z := make([]zone, n[NZone])
	for i := 0; i < len(z); i++ {
		var ok bool
		var n uint32
		if n, ok = zonedata.big4(); !ok {
			return nil, false
		}
		z[i].utcoff = int(n)
		var b byte
		if b, ok = zonedata.byte(); !ok {
			return nil, false
		}
		z[i].isdst = b != 0
		if b, ok = zonedata.byte(); !ok || int(b) >= len(abbrev) {
			return nil, false
		}
		z[i].name = byteString(abbrev[b:])
	}

	// Now the transition time info.
	zt = make([]zonetime, n[NTime])
	for i := 0; i < len(zt); i++ {
		var ok bool
		var n uint32
		if n, ok = txtimes.big4(); !ok {
			return nil, false
		}
		zt[i].time = int32(n)
		if int(txzones[i]) >= len(z) {
			return nil, false
		}
		zt[i].zone = &z[txzones[i]]
		if i < len(isstd) {
			zt[i].isstd = isstd[i] != 0
		}
		if i < len(isutc) {
			zt[i].isutc = isutc[i] != 0
		}
	}
	return zt, true
}

func readinfofile(name string) ([]zonetime, bool) {
	buf, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, false
	}
	return parseinfo(buf)
}

var zones []zonetime
var onceSetupZone sync.Once

func setupZone() {
	// consult $TZ to find the time zone to use.
	// no $TZ means use the system default /etc/localtime.
	// $TZ="" means use UTC.
	// $TZ="foo" means use /usr/share/zoneinfo/foo.
	// Many systems use /usr/share/zoneinfo, Solaris 2 has
	// /usr/share/lib/zoneinfo, IRIX 6 has /usr/lib/locale/TZ.
	zoneDirs := []string{"/usr/share/zoneinfo/",
		"/usr/share/lib/zoneinfo/",
		"/usr/lib/locale/TZ/"}

	tz, err := os.Getenverror("TZ")
	switch {
	case err == os.ENOENV:
		zones, _ = readinfofile("/etc/localtime")
	case len(tz) > 0:
		for _, zoneDir := range zoneDirs {
			var ok bool
			if zones, ok = readinfofile(zoneDir + tz); ok {
				break
			}
		}
	case len(tz) == 0:
		// do nothing: use UTC
	}
}

// Look up the correct time zone (daylight savings or not) for the given unix time, in the current location.
func lookupTimezone(sec int64) (zone string, offset int) {
	onceSetupZone.Do(setupZone)
	if len(zones) == 0 {
		return "UTC", 0
	}

	// Binary search for entry with largest time <= sec
	tz := zones
	for len(tz) > 1 {
		m := len(tz) / 2
		if sec < int64(tz[m].time) {
			tz = tz[0:m]
		} else {
			tz = tz[m:]
		}
	}
	z := tz[0].zone
	return z.name, z.utcoff
}

// lookupByName returns the time offset for the
// time zone with the given abbreviation. It only considers
// time zones that apply to the current system.
// For example, for a system configured as being in New York,
// it only recognizes "EST" and "EDT".
// For a system in San Francisco, "PST" and "PDT".
// For a system in Sydney, "EST" and "EDT", though they have
// different meanings than they do in New York.
func lookupByName(name string) (off int, found bool) {
	onceSetupZone.Do(setupZone)
	for _, z := range zones {
		if name == z.zone.name {
			return z.zone.utcoff, true
		}
	}
	return 0, false
}
<|endoftext|>


<commit_before>package naming

import (
	"net"
	"time"

	"v.io/core/veyron2/context"

	verror "v.io/core/veyron2/verror2"
)

const (
	pkgPath         = "v.io/core/veyron2/naming"
	UnknownProtocol = ""
)

var (
	ErrNameExists              = verror.Register(pkgPath+".nameExists", verror.NoRetry, "{1} {2} Name exists {_}")
	ErrNoSuchName              = verror.Register(pkgPath+".nameDoesntExist", verror.NoRetry, "{1} {2} Name {3} doesn't exist {_}")
	ErrNoSuchNameRoot          = verror.Register(pkgPath+".rootNameDoesntExist", verror.NoRetry, "{1} {2} Namespace root name {3} doesn't exist {_}")
	ErrResolutionDepthExceeded = verror.Register(pkgPath+".resolutionDepthExceeded", verror.NoRetry, "{1} {2} Resolution depth exceeded {_}")
	ErrNoMountTable            = verror.Register(pkgPath+".noMounttable", verror.NoRetry, "{1} {2} No mounttable {_}")
)

// Endpoint represents unique identifiers for entities communicating over a
// network.  End users don't use endpoints - they deal solely with object names,
// with the MountTable providing translation of object names to endpoints.
type Endpoint interface {
	// Network returns "veyron" so that Endpoint can implement net.Addr.
	Network() string

	// String returns a string representation of the endpoint.
	//
	// The String method formats the endpoint as:
	//   @<version>@<version specific fields>@@
	// Where version is an unsigned integer.
	//
	// Version 1 is the current version for network address information:
	//   @1@<protocol>@<address>@<routingid>@@
	// Where protocol is the underlying network protocol (tcp, bluetooth etc.)
	// and address is the address specific to that protocol (host:port for
	// tcp, MAC address for bluetooth etc.)
	//
	// Version 2 is the previous version for RPC:
	//   @2@<protocol>@<address>@<routingid>@<rpc version>@<rpc codec>@@
	//
	// Version 3 is the current version for RPC:
	//   @3@<protocol>@<address>@<routingid>@<rpc version>@<rpc codec>@m|s@@
	//
	// Along with Network, this method ensures that Endpoint implements net.Addr.
	String() string

	// Name returns a string reprsentation of this Endpoint that can
	// be used as a name with ipc.StartCall.
	Name() string

	// VersionedString returns a string in the specified format. If the version
	// number is unsupported, the current 'default' version will be used.
	VersionedString(version int) string

	// RoutingID returns the RoutingID associated with this Endpoint.
	RoutingID() RoutingID

	// Addrs returns a net.Addr whose String method will return the
	// the underlying network address encoded in the endpoint rather than
	// the endpoint string itself.
	// For example, for TCP based endpoints it will return a net.Addr
	// whose network is "tcp" and string representation is <host>:<port>,
	// than the full Veyron endpoint as per the String method above.
	Addr() net.Addr

	// ServesMountTable returns true if this endpoint serves a mount table.
	ServesMountTable() bool
}

// MountedServer represents a server mounted under an object name.
//
// TODO(toddw): Consolidate with VDLMountedServer once vdl supports time.
type MountedServer struct {
	Server           string    // Server is an object address (OA): endpoint + suffix
	Expires          time.Time // Absolute time after which the mount expires.
	BlessingPatterns []string  // Patterns matching blessings presented by Server.
}

// MountEntry represents a name mounted in the mounttable.
//
// TODO(toddw): Consolidate with VDLMountEntry once vdl supports time.
type MountEntry struct {
	// Name is the mounted name.
	Name string
	// Servers (if present) specifies the mounted names.
	Servers []MountedServer
	// mt is true if servers refer to another mount table.
	mt bool
	// Pattern is a security.BlessingPattern that should match the servers.
	Pattern string
	// An error occurred fulfilling the request.
	Error error
}

// ServesMountTable returns true if the mount entry represents servers that are
// mount tables.
// TODO(p): When the endpoint actually has this fact encoded in, use that.
func (e *MountEntry) ServesMountTable() bool { return e.mt }

// SetServesMountTable sets whether or not this is a mount table.
func (e *MountEntry) SetServesMountTable(v bool) { e.mt = v }

// Names returns the servers represented by MountEntry as names, including
// the MountedName suffix.
func (e *MountEntry) Names() []string {
	var names []string
	for _, s := range e.Servers {
		names = append(names, JoinAddressName(s.Server, e.Name))
	}
	return names
}

// CacheCtl is a cache control for the resolution cache.
type CacheCtl interface {
	CacheCtl()
}

// DisbleCache disables the resolution cache when set to true and enables if false.
// As a side effect one can flush the cache by disabling and then reenabling it.
type DisableCache bool

func (DisableCache) CacheCtl() {}

// MountOpt is the interface for all Mount options.
type MountOpt interface {
	NSMountOpt()
}

// ReplaceMountOpt requests the mount to replace the previous mount.
type ReplaceMountOpt bool

func (ReplaceMountOpt) NSMountOpt() {}

// ServesMountTableOpt means the target is a mount table.
type ServesMountTableOpt bool

func (ServesMountTableOpt) NSMountOpt()  {}
func (ServesMountTableOpt) EndpointOpt() {}

// ResolveOpt is the interface for all Mount options.
type ResolveOpt interface {
	NSResolveOpt()
}

// RootBlessingPatternOpt specifies a blessing pattern that the root
// mount table must match.
type RootBlessingPatternOpt string

func (RootBlessingPatternOpt) NSResolveOpt() {}

// TODO(p): Perhaps add an ACL Opt.

// Namespace provides translation from object names to server object addresses.
// It represents the interface to a client side library for the MountTable
// service
type Namespace interface {
	// Mount the server object address under the object name, expiring after
	// the ttl. ttl of zero implies an implementation-specific high value
	// (essentially, forever).
	Mount(ctx *context.T, name, server string, ttl time.Duration, opts ...MountOpt) error

	// Unmount the server object address from the object name, or if server
	// is empty, unmount all server OAs from the object name.
	Unmount(ctx *context.T, name, server string) error

	// Resolve the object name into its mounted servers.
	Resolve(ctx *context.T, name string, opts ...ResolveOpt) (entry *MountEntry, err error)

	// ResolveToMountTable resolves the object name into the mounttables
	// directly responsible for the name.
	ResolveToMountTable(ctx *context.T, name string, opts ...ResolveOpt) (entry *MountEntry, err error)

	// FlushCacheEntry flushes resolution information cached for the name.  If
	// anything was flushed it returns true.
	FlushCacheEntry(name string) bool

	// CacheCtl sets controls and returns the current control values.
	CacheCtl(ctls ...CacheCtl) []CacheCtl

	// Glob returns all names matching pattern.  If recursive is true, it also
	// returns all names below the matching ones.
	Glob(ctx *context.T, pattern string) (chan MountEntry, error)

	// SetRoots sets the roots that the local Namespace is
	// relative to. All relative names passed to the methods above
	// will be interpreted as relative to these roots. The roots
	// will be tried in the order that they are specified in the parameter
	// list for SetRoots. Calling SetRoots with no arguments will clear the
	// currently configured set of roots.
	SetRoots(roots ...string) error

	// Roots returns the currently configured roots. An empty slice is
	// returned if no roots are configured.
	Roots() []string
}

// When this prefix is present at the beginning of an object name suffix, the
// server may intercept the request and handle it internally. This is used to
// provide debugging, monitoring and other common functionality across all
// servers. Applications cannot use any name component that starts with this
// prefix.
const ReservedNamePrefix = "__"
<commit_msg>namespace: Glob now returns a chan of interface{} where the possible returned types can be *naming.GlobError or *naming.MountEntry.  I have no idea why anyone things this is better than just having an error field in MountEntry but apparently everyone else does.<commit_after>package naming

import (
	"net"
	"time"

	"v.io/core/veyron2/context"

	verror "v.io/core/veyron2/verror2"
)

const (
	pkgPath         = "v.io/core/veyron2/naming"
	UnknownProtocol = ""
)

var (
	ErrNameExists              = verror.Register(pkgPath+".nameExists", verror.NoRetry, "{1} {2} Name exists {_}")
	ErrNoSuchName              = verror.Register(pkgPath+".nameDoesntExist", verror.NoRetry, "{1} {2} Name {3} doesn't exist {_}")
	ErrNoSuchNameRoot          = verror.Register(pkgPath+".rootNameDoesntExist", verror.NoRetry, "{1} {2} Namespace root name {3} doesn't exist {_}")
	ErrResolutionDepthExceeded = verror.Register(pkgPath+".resolutionDepthExceeded", verror.NoRetry, "{1} {2} Resolution depth exceeded {_}")
	ErrNoMountTable            = verror.Register(pkgPath+".noMounttable", verror.NoRetry, "{1} {2} No mounttable {_}")
)

// Endpoint represents unique identifiers for entities communicating over a
// network.  End users don't use endpoints - they deal solely with object names,
// with the MountTable providing translation of object names to endpoints.
type Endpoint interface {
	// Network returns "veyron" so that Endpoint can implement net.Addr.
	Network() string

	// String returns a string representation of the endpoint.
	//
	// The String method formats the endpoint as:
	//   @<version>@<version specific fields>@@
	// Where version is an unsigned integer.
	//
	// Version 1 is the current version for network address information:
	//   @1@<protocol>@<address>@<routingid>@@
	// Where protocol is the underlying network protocol (tcp, bluetooth etc.)
	// and address is the address specific to that protocol (host:port for
	// tcp, MAC address for bluetooth etc.)
	//
	// Version 2 is the previous version for RPC:
	//   @2@<protocol>@<address>@<routingid>@<rpc version>@<rpc codec>@@
	//
	// Version 3 is the current version for RPC:
	//   @3@<protocol>@<address>@<routingid>@<rpc version>@<rpc codec>@m|s@@
	//
	// Along with Network, this method ensures that Endpoint implements net.Addr.
	String() string

	// Name returns a string reprsentation of this Endpoint that can
	// be used as a name with ipc.StartCall.
	Name() string

	// VersionedString returns a string in the specified format. If the version
	// number is unsupported, the current 'default' version will be used.
	VersionedString(version int) string

	// RoutingID returns the RoutingID associated with this Endpoint.
	RoutingID() RoutingID

	// Addrs returns a net.Addr whose String method will return the
	// the underlying network address encoded in the endpoint rather than
	// the endpoint string itself.
	// For example, for TCP based endpoints it will return a net.Addr
	// whose network is "tcp" and string representation is <host>:<port>,
	// than the full Veyron endpoint as per the String method above.
	Addr() net.Addr

	// ServesMountTable returns true if this endpoint serves a mount table.
	ServesMountTable() bool
}

// MountedServer represents a server mounted under an object name.
//
// TODO(toddw): Consolidate with VDLMountedServer once vdl supports time.
type MountedServer struct {
	Server           string    // Server is an object address (OA): endpoint + suffix
	Expires          time.Time // Absolute time after which the mount expires.
	BlessingPatterns []string  // Patterns matching blessings presented by Server.
}

// MountEntry represents a name mounted in the mounttable.
//
// TODO(toddw): Consolidate with VDLMountEntry once vdl supports time.
type MountEntry struct {
	// Name is the mounted name.
	Name string
	// Servers (if present) specifies the mounted names.
	Servers []MountedServer
	// mt is true if servers refer to another mount table.
	mt bool
	// Pattern is a security.BlessingPattern that should match the servers.
	Pattern string
}

// GlobError is returned by namespace.Glob to indicate a piece of the namespace
// that could not be traversed.
type GlobError struct {
	// Name is the mounted name.
	Name string
	// An error occurred fulfilling the request.
	Error error
}

// ServesMountTable returns true if the mount entry represents servers that are
// mount tables.
// TODO(p): When the endpoint actually has this fact encoded in, use that.
func (e *MountEntry) ServesMountTable() bool { return e.mt }

// SetServesMountTable sets whether or not this is a mount table.
func (e *MountEntry) SetServesMountTable(v bool) { e.mt = v }

// Names returns the servers represented by MountEntry as names, including
// the MountedName suffix.
func (e *MountEntry) Names() []string {
	var names []string
	for _, s := range e.Servers {
		names = append(names, JoinAddressName(s.Server, e.Name))
	}
	return names
}

// CacheCtl is a cache control for the resolution cache.
type CacheCtl interface {
	CacheCtl()
}

// DisbleCache disables the resolution cache when set to true and enables if false.
// As a side effect one can flush the cache by disabling and then reenabling it.
type DisableCache bool

func (DisableCache) CacheCtl() {}

// MountOpt is the interface for all Mount options.
type MountOpt interface {
	NSMountOpt()
}

// ReplaceMountOpt requests the mount to replace the previous mount.
type ReplaceMountOpt bool

func (ReplaceMountOpt) NSMountOpt() {}

// ServesMountTableOpt means the target is a mount table.
type ServesMountTableOpt bool

func (ServesMountTableOpt) NSMountOpt()  {}
func (ServesMountTableOpt) EndpointOpt() {}

// ResolveOpt is the interface for all Mount options.
type ResolveOpt interface {
	NSResolveOpt()
}

// RootBlessingPatternOpt specifies a blessing pattern that the root
// mount table must match.
type RootBlessingPatternOpt string

func (RootBlessingPatternOpt) NSResolveOpt() {}

// TODO(p): Perhaps add an ACL Opt.

// Namespace provides translation from object names to server object addresses.
// It represents the interface to a client side library for the MountTable
// service
type Namespace interface {
	// Mount the server object address under the object name, expiring after
	// the ttl. ttl of zero implies an implementation-specific high value
	// (essentially, forever).
	Mount(ctx *context.T, name, server string, ttl time.Duration, opts ...MountOpt) error

	// Unmount the server object address from the object name, or if server
	// is empty, unmount all server OAs from the object name.
	Unmount(ctx *context.T, name, server string) error

	// Resolve the object name into its mounted servers.
	Resolve(ctx *context.T, name string, opts ...ResolveOpt) (entry *MountEntry, err error)

	// ResolveToMountTable resolves the object name into the mounttables
	// directly responsible for the name.
	ResolveToMountTable(ctx *context.T, name string, opts ...ResolveOpt) (entry *MountEntry, err error)

	// FlushCacheEntry flushes resolution information cached for the name.  If
	// anything was flushed it returns true.
	FlushCacheEntry(name string) bool

	// CacheCtl sets controls and returns the current control values.
	CacheCtl(ctls ...CacheCtl) []CacheCtl

	// Glob returns MountEntry's whose name matches the pattern and GlobError's
	// for any piece of the space that can't be traversed.
	//
	// Two special patterns:
	//   prefix/... means all names below prefix.
	//   prefix/*** is like prefix/... but doesn't traverse into non-mounttable servers.
	//
	// Example:
	//	rc, err := ns.Glob(ctx, pattern)
	//	if err != nil {
	//		boom(t, "Glob(%s): %s", pattern, err)
	//	}
	//	for s := range rc {
	//		switch v := s.(type) {
	//		case *naming.MountEntry:
	//			fmt.Printf("%s: %v\n", v.Name, v.Servers)
	//		case *naming.GlobError:
	//			fmt.Fprintf(stderr, "%s can't be traversed: %s\n", v.Name, v.Error)
	//		}
	//	}
	Glob(ctx *context.T, pattern string) (chan interface{}, error)

	// SetRoots sets the roots that the local Namespace is
	// relative to. All relative names passed to the methods above
	// will be interpreted as relative to these roots. The roots
	// will be tried in the order that they are specified in the parameter
	// list for SetRoots. Calling SetRoots with no arguments will clear the
	// currently configured set of roots.
	SetRoots(roots ...string) error

	// Roots returns the currently configured roots. An empty slice is
	// returned if no roots are configured.
	Roots() []string
}

// When this prefix is present at the beginning of an object name suffix, the
// server may intercept the request and handle it internally. This is used to
// provide debugging, monitoring and other common functionality across all
// servers. Applications cannot use any name component that starts with this
// prefix.
const ReservedNamePrefix = "__"
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/rs/cors"

	"github.com/ericsage/deepcell/dc"

	"google.golang.org/grpc"
)

type Response struct {
	Data   *dc.Reply `json:"data"`
	Errors []string  `json:"errors"`
}

var (
	tls        = getenv("tls", "false")
	serverAddr = getenv("SERVER_ADDRESS", "biancone.ucsd.edu")
	serverPort = getenv("SERVER_PORT", "5000")
)

func getenv(key, fallback string) string {
	value := os.Getenv(key)
	if len(value) == 0 {
		return fallback
	}
	return value
}

func knockout(ontology string, genes []string) (*dc.Reply, error) {
	log.Println("About to open connection")
	address := serverAddr + ":" + serverPort
	log.Println(address)
	conn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	log.Println("Dialed server")
	defer conn.Close()
	client := dc.NewDeepCellClient(conn)
	req := &dc.Request{
		Genes:    genes,
		Ontology: ontology,
		Growth:   false,
	}
	log.Println("About to process request")
	rep, err := client.Run(context.Background(), req)
	if err != nil {
		return nil, err
	}
	return rep, nil
}

func deepcellHandler(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Type", "application/json")
	//// CORS support
	w.Header().Set("Access-Control-Allow-Origin", "*")

	defer catchError()
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		panic("Could not read request body!")
	}

	url := r.URL
	ontology := url.Query().Get("ontology")
	if ontology == "" {
		ontology = "GO"
	}
	var genes []string
	err = json.Unmarshal(body, &genes)
	if err != nil {
		panic("Could not decode json into genes list!")
	}
	termReply, err := knockout(ontology, genes)
	errors := []string{}
	if err != nil {
		errors = []string{ err.Error() }
		termReply = ""
	}
	res := &Response{
		Data:   termReply,
		Errors: errors,
	}
	json.NewEncoder(w).Encode(res)
}

func catchError() {
	if r := recover(); r != nil {
		//Send response json error response here
		log.Println(r)

	}
}

func main() {
	log.Println("Starting server...")
	mux := http.NewServeMux()
	handler := cors.Default().Handler(mux)

	mux.HandleFunc("/", deepcellHandler)
	http.ListenAndServe(":8888", handler)
}
<commit_msg>Update main.go<commit_after>package main

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/rs/cors"

	"github.com/ericsage/deepcell/dc"

	"google.golang.org/grpc"
)

type Response struct {
	Data   *dc.Reply `json:"data"`
	Errors []string  `json:"errors"`
}

var (
	tls        = getenv("tls", "false")
	serverAddr = getenv("SERVER_ADDRESS", "biancone.ucsd.edu")
	serverPort = getenv("SERVER_PORT", "5000")
)

func getenv(key, fallback string) string {
	value := os.Getenv(key)
	if len(value) == 0 {
		return fallback
	}
	return value
}

func knockout(ontology string, genes []string) (*dc.Reply, error) {
	log.Println("About to open connection")
	address := serverAddr + ":" + serverPort
	log.Println(address)
	conn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	log.Println("Dialed server")
	defer conn.Close()
	client := dc.NewDeepCellClient(conn)
	req := &dc.Request{
		Genes:    genes,
		Ontology: ontology,
		Growth:   false,
	}
	log.Println("About to process request")
	rep, err := client.Run(context.Background(), req)
	if err != nil {
		return nil, err
	}
	return rep, nil
}

func deepcellHandler(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Type", "application/json")
	//// CORS support
	w.Header().Set("Access-Control-Allow-Origin", "*")

	defer catchError()
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		panic("Could not read request body!")
	}

	url := r.URL
	ontology := url.Query().Get("ontology")
	if ontology == "" {
		ontology = "GO"
	}
	var genes []string
	err = json.Unmarshal(body, &genes)
	if err != nil {
		panic("Could not decode json into genes list!")
	}
	termReply, err := knockout(ontology, genes)
	errors := []string{}
	if err != nil {
		errors = []string{ err.Error() }
		termReply = &dc.Reply{}
	}
	res := &Response{
		Data:   termReply,
		Errors: errors,
	}
	json.NewEncoder(w).Encode(res)
}

func catchError() {
	if r := recover(); r != nil {
		//Send response json error response here
		log.Println(r)

	}
}

func main() {
	log.Println("Starting server...")
	mux := http.NewServeMux()
	handler := cors.Default().Handler(mux)

	mux.HandleFunc("/", deepcellHandler)
	http.ListenAndServe(":8888", handler)
}
<|endoftext|>