<commit_before>package scheduler

import (
	"container/list"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/swarm-v2/api"
	"github.com/docker/swarm-v2/manager/state"
)

// Scheduler assigns tasks to nodes.
type Scheduler struct {
	store           state.WatchableStore
	unassignedTasks *list.List

	// stopChan signals to the state machine to stop running
	stopChan chan struct{}
	// doneChan is closed when the state machine terminates
	doneChan chan struct{}
}

// New creates a new scheduler.
func New(store state.WatchableStore) *Scheduler {
	return &Scheduler{
		store:           store,
		unassignedTasks: list.New(),
		stopChan:        make(chan struct{}),
		doneChan:        make(chan struct{}),
	}
}

func (s *Scheduler) setupTasksList(tx state.ReadTx) error {
	tasks, err := tx.Tasks().Find(state.All)
	if err != nil {
		return err
	}
	for _, t := range tasks {
		if t.NodeID == "" {
			log.Infof("Queueing %#v", t)
			s.enqueue(t)
		}
	}

	return nil
}

// Run is the scheduler event loop.
func (s *Scheduler) Run() error {
	defer close(s.doneChan)

	updates := s.store.WatchQueue().Watch()
	defer s.store.WatchQueue().StopWatch(updates)

	err := s.store.View(s.setupTasksList)
	if err != nil {
		log.Errorf("could not snapshot store: %v", err)
		return err
	}

	// Queue all unassigned tasks before processing changes.
	s.tick()

	pendingChanges := 0

	// Watch for changes.
	for {
		select {
		case event := <-updates:
			switch v := event.Payload.(type) {
			case state.EventCreateTask:
				pendingChanges += s.createTask(v.Task)
			case state.EventUpdateTask:
				pendingChanges += s.createTask(v.Task)
			case state.EventCreateNode:
				if v.Node.Status.State == api.NodeStatus_READY {
					pendingChanges++
				}
			case state.EventUpdateNode:
				if v.Node.Status.State == api.NodeStatus_READY {
					pendingChanges++
				}
			case state.EventCommit:
				if pendingChanges > 0 {
					s.tick()
					pendingChanges = 0
				}
			}
		case <-s.stopChan:
			return nil
		}
	}
}

// Stop causes the scheduler event loop to stop running.
func (s *Scheduler) Stop() {
	close(s.stopChan)
	<-s.doneChan
}

// enqueue queues a task for scheduling.
func (s *Scheduler) enqueue(t *api.Task) {
	s.unassignedTasks.PushBack(t)
}

func (s *Scheduler) createTask(t *api.Task) int {
	if t.NodeID == "" {
		// unassigned task
		s.enqueue(t)
		return 1
	}
	return 0
}

// tick attempts to schedule the queue.
func (s *Scheduler) tick() {
	nextBatch := list.New()

	// TODO(aaronl): Ideally, we would make scheduling decisions outside
	// of an Update callback, since Update blocks other writes to the
	// store. The current approach of making the decisions inside Update
	// is done to keep the store simple. Eventually, we may want to break
	// this up into a View where the decisions are made, and an Update that
	// applies them. This will require keeping local state to keep track of
	// allocations as they are made, since the store itself can't be
	// changed through View.
	err := s.store.Update(func(tx state.Tx) error {
		nodes, err := tx.Nodes().Find(state.All)
		if err != nil {
			return err
		}

		var next *list.Element
		for e := s.unassignedTasks.Front(); e != nil; e = next {
			next = e.Next()
			t := e.Value.(*api.Task)
			if newT := s.scheduleTask(tx, nodes, *t); newT == nil {
				// scheduling failed; keep this task in the list
				nextBatch.PushBack(t)
			}
		}
		return nil
	})

	if err != nil {
		log.Errorf("Error in transaction: %v", err)

		// leave unassignedTasks list in place
	} else {
		s.unassignedTasks = nextBatch
	}
}

// scheduleTask schedules a single task.
func (s *Scheduler) scheduleTask(tx state.Tx, nodes []*api.Node, t api.Task) *api.Task {
	node := s.selectNodeForTask(tx, nodes, &t)
	if node == nil {
		log.Info("No nodes available to assign tasks to")
		return nil
	}

	log.Infof("Assigning task %s to node %s", t.ID, node.ID)
	t.NodeID = node.ID
	t.Status = &api.TaskStatus{State: api.TaskStateAssigned}
	if err := tx.Tasks().Update(&t); err != nil {
		log.Error(err)
		return nil
	}
	return &t
}

// selectNodeForTask is a naive scheduler. Will select a ready, non-drained
// node with the fewer number of tasks already running.
func (s *Scheduler) selectNodeForTask(tx state.Tx, nodes []*api.Node, t *api.Task) *api.Node {
	var target *api.Node
	targetTasks := 0

	for _, n := range nodes {
		if n.Status.State != api.NodeStatus_READY || (n.Spec != nil && n.Spec.Availability != api.NodeAvailabilityActive) {
			continue
		}

		tasks, err := tx.Tasks().Find(state.ByNodeID(n.ID))
		if err != nil {
			log.Errorf("Error selecting tasks by node: %v", err)
			continue
		}
		nodeTasks := len(tasks)
		if target == nil || nodeTasks < targetTasks {
			target = n
			targetTasks = nodeTasks
		}
	}

	return target
}
<commit_msg>scheduler: Reduce log output<commit_after>package scheduler

import (
	"container/list"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/swarm-v2/api"
	"github.com/docker/swarm-v2/manager/state"
)

// Scheduler assigns tasks to nodes.
type Scheduler struct {
	store           state.WatchableStore
	unassignedTasks *list.List

	// stopChan signals to the state machine to stop running
	stopChan chan struct{}
	// doneChan is closed when the state machine terminates
	doneChan chan struct{}
}

// New creates a new scheduler.
func New(store state.WatchableStore) *Scheduler {
	return &Scheduler{
		store:           store,
		unassignedTasks: list.New(),
		stopChan:        make(chan struct{}),
		doneChan:        make(chan struct{}),
	}
}

func (s *Scheduler) setupTasksList(tx state.ReadTx) error {
	tasks, err := tx.Tasks().Find(state.All)
	if err != nil {
		return err
	}
	for _, t := range tasks {
		if t.NodeID == "" {
			s.enqueue(t)
		}
	}

	return nil
}

// Run is the scheduler event loop.
func (s *Scheduler) Run() error {
	defer close(s.doneChan)

	updates := s.store.WatchQueue().Watch()
	defer s.store.WatchQueue().StopWatch(updates)

	err := s.store.View(s.setupTasksList)
	if err != nil {
		log.Errorf("could not snapshot store: %v", err)
		return err
	}

	// Queue all unassigned tasks before processing changes.
	s.tick()

	pendingChanges := 0

	// Watch for changes.
	for {
		select {
		case event := <-updates:
			switch v := event.Payload.(type) {
			case state.EventCreateTask:
				pendingChanges += s.createTask(v.Task)
			case state.EventUpdateTask:
				pendingChanges += s.createTask(v.Task)
			case state.EventCreateNode:
				if v.Node.Status.State == api.NodeStatus_READY {
					pendingChanges++
				}
			case state.EventUpdateNode:
				if v.Node.Status.State == api.NodeStatus_READY {
					pendingChanges++
				}
			case state.EventCommit:
				if pendingChanges > 0 {
					s.tick()
					pendingChanges = 0
				}
			}
		case <-s.stopChan:
			return nil
		}
	}
}

// Stop causes the scheduler event loop to stop running.
func (s *Scheduler) Stop() {
	close(s.stopChan)
	<-s.doneChan
}

// enqueue queues a task for scheduling.
func (s *Scheduler) enqueue(t *api.Task) {
	s.unassignedTasks.PushBack(t)
}

func (s *Scheduler) createTask(t *api.Task) int {
	if t.NodeID == "" {
		// unassigned task
		s.enqueue(t)
		return 1
	}
	return 0
}

// tick attempts to schedule the queue.
func (s *Scheduler) tick() {
	nextBatch := list.New()

	// TODO(aaronl): Ideally, we would make scheduling decisions outside
	// of an Update callback, since Update blocks other writes to the
	// store. The current approach of making the decisions inside Update
	// is done to keep the store simple. Eventually, we may want to break
	// this up into a View where the decisions are made, and an Update that
	// applies them. This will require keeping local state to keep track of
	// allocations as they are made, since the store itself can't be
	// changed through View.
	err := s.store.Update(func(tx state.Tx) error {
		nodes, err := tx.Nodes().Find(state.All)
		if err != nil {
			return err
		}

		var next *list.Element
		for e := s.unassignedTasks.Front(); e != nil; e = next {
			next = e.Next()
			t := e.Value.(*api.Task)
			if newT := s.scheduleTask(tx, nodes, *t); newT == nil {
				// scheduling failed; keep this task in the list
				nextBatch.PushBack(t)
			}
		}
		return nil
	})

	if err != nil {
		log.Errorf("Error in transaction: %v", err)

		// leave unassignedTasks list in place
	} else {
		s.unassignedTasks = nextBatch
	}
}

// scheduleTask schedules a single task.
func (s *Scheduler) scheduleTask(tx state.Tx, nodes []*api.Node, t api.Task) *api.Task {
	node := s.selectNodeForTask(tx, nodes, &t)
	if node == nil {
		log.WithField("task.id", t.ID).Debug("No nodes available to assign tasks to")
		return nil
	}

	log.WithField("task.id", t.ID).Debugf("Assigning to node %s", node.ID)
	t.NodeID = node.ID
	t.Status = &api.TaskStatus{State: api.TaskStateAssigned}
	if err := tx.Tasks().Update(&t); err != nil {
		log.Error(err)
		return nil
	}
	return &t
}

// selectNodeForTask is a naive scheduler. Will select a ready, non-drained
// node with the fewer number of tasks already running.
func (s *Scheduler) selectNodeForTask(tx state.Tx, nodes []*api.Node, t *api.Task) *api.Node {
	var target *api.Node
	targetTasks := 0

	for _, n := range nodes {
		if n.Status.State != api.NodeStatus_READY || (n.Spec != nil && n.Spec.Availability != api.NodeAvailabilityActive) {
			continue
		}

		tasks, err := tx.Tasks().Find(state.ByNodeID(n.ID))
		if err != nil {
			log.Errorf("Error selecting tasks by node: %v", err)
			continue
		}
		nodeTasks := len(tasks)
		if target == nil || nodeTasks < targetTasks {
			target = n
			targetTasks = nodeTasks
		}
	}

	return target
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This program takes an HTML file and outputs a corresponding article file in
// present format. See: code.google.com/p/go.tools/present
package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"strings"

	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
)

func main() {
	flag.Parse()

	err := convert(os.Stdout, os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
}

func convert(w io.Writer, r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}

	style := find(root, isTag(atom.Style))
	parseStyles(style)

	body := find(root, isTag(atom.Body))
	if body == nil {
		return errors.New("couldn't find body")
	}
	article := limitNewlineRuns(makeHeadings(strings.TrimSpace(text(body))))
	_, err = fmt.Fprintf(w, "Title\n\n%s", article)
	return err
}

type Style string

const (
	Bold   Style = "*"
	Italic Style = "_"
	Code   Style = "`"
)

var cssRules = make(map[string]Style)

func parseStyles(style *html.Node) {
	if style == nil || style.FirstChild == nil {
		log.Println("couldn't find styles")
		return
	}
	s := bufio.NewScanner(strings.NewReader(style.FirstChild.Data))

	findRule := func(b []byte, atEOF bool) (advance int, token []byte, err error) {
		if i := bytes.Index(b, []byte("{")); i >= 0 {
			token = bytes.TrimSpace(b[:i])
			advance = i
		}
		return
	}
	findBody := func(b []byte, atEOF bool) (advance int, token []byte, err error) {
		if len(b) == 0 {
			return
		}
		if b[0] != '{' {
			err = fmt.Errorf("expected {, got %c", b[0])
			return
		}
		if i := bytes.Index(b, []byte("}")); i < 0 {
			err = fmt.Errorf("can't find closing }")
			return
		} else {
			token = b[1:i]
			advance = i + 1
		}
		return
	}

	s.Split(findRule)
	for s.Scan() {
		rule := s.Text()
		s.Split(findBody)
		if !s.Scan() {
			break
		}
		b := strings.ToLower(s.Text())
		switch {
		case strings.Contains(b, "italic"):
			cssRules[rule] = Italic
		case strings.Contains(b, "bold"):
			cssRules[rule] = Bold
		case strings.Contains(b, "Consolas") || strings.Contains(b, "Courier New"):
			cssRules[rule] = Code
		}
		s.Split(findRule)
	}
	if err := s.Err(); err != nil {
		log.Println(err)
	}
}

var newlineRun = regexp.MustCompile(`\n\n+`)

func limitNewlineRuns(s string) string {
	return newlineRun.ReplaceAllString(s, "\n\n")
}

func makeHeadings(body string) string {
	buf := new(bytes.Buffer)
	lines := strings.Split(body, "\n")
	for i, s := range lines {
		if i == 0 && !isBoldTitle(s) {
			buf.WriteString("* Introduction\n\n")
		}
		if isBoldTitle(s) {
			s = strings.TrimSpace(strings.Replace(s, "*", " ", -1))
			s = "* " + s
		}
		buf.WriteString(s)
		buf.WriteByte('\n')
	}
	return buf.String()
}

func isBoldTitle(s string) bool {
	return !strings.Contains(s, " ") &&
		strings.HasPrefix(s, "*") &&
		strings.HasSuffix(s, "*")
}

func indent(buf *bytes.Buffer, s string) {
	for _, l := range strings.Split(s, "\n") {
		if l != "" {
			buf.WriteByte('\t')
			buf.WriteString(l)
		}
		buf.WriteByte('\n')
	}
}

func unwrap(buf *bytes.Buffer, s string) {
	var cont bool
	for _, l := range strings.Split(s, "\n") {
		l = strings.TrimSpace(l)
		if len(l) == 0 {
			if cont {
				buf.WriteByte('\n')
				buf.WriteByte('\n')
			}
			cont = false
		} else {
			if cont {
				buf.WriteByte(' ')
			}
			buf.WriteString(l)
			cont = true
		}
	}
}

func text(n *html.Node) string {
	var buf bytes.Buffer
	walk(n, func(n *html.Node) bool {
		switch n.Type {
		case html.TextNode:
			buf.WriteString(n.Data)
			return false
		case html.ElementNode:
			// no-op
		default:
			return true
		}
		a := n.DataAtom
		if a == atom.Span {
			switch {
			case hasStyle(Code)(n):
				a = atom.Code
			case hasStyle(Bold)(n):
				a = atom.B
			case hasStyle(Italic)(n):
				a = atom.I
			}
		}
		switch a {
		case atom.Br:
			buf.WriteByte('\n')
		case atom.P:
			unwrap(&buf, childText(n))
			buf.WriteString("\n\n")
		case atom.Li:
			buf.WriteString("- ")
			unwrap(&buf, childText(n))
			buf.WriteByte('\n')
		case atom.Pre:
			indent(&buf, childText(n))
			buf.WriteByte('\n')
		case atom.A:
			fmt.Fprintf(&buf, "[[%s][%s]]", attr(n, "href"), childText(n))
		case atom.Code:
			buf.WriteString(highlight(n, "`"))
		case atom.B:
			buf.WriteString(highlight(n, "*"))
		case atom.I:
			buf.WriteString(highlight(n, "_"))
		case atom.Img:
			src := attr(n, "src")
			fmt.Fprintf(&buf, ".image %s\n", src)
		case atom.Iframe:
			src, w, h := attr(n, "src"), attr(n, "width"), attr(n, "height")
			fmt.Fprintf(&buf, "\n.iframe %s %s %s\n", src, h, w)
		case atom.Param:
			if attr(n, "name") == "movie" {
				// Old style YouTube embed.
				u := attr(n, "value")
				u = strings.Replace(u, "/v/", "/embed/", 1)
				if i := strings.Index(u, "&"); i >= 0 {
					u = u[:i]
				}
				fmt.Fprintf(&buf, "\n.iframe %s 540 304\n", u)
			}
		default:
			return true
		}
		return false
	})
	return buf.String()
}

func childText(node *html.Node) string {
	var buf bytes.Buffer
	for n := node.FirstChild; n != nil; n = n.NextSibling {
		fmt.Fprint(&buf, text(n))
	}
	return buf.String()
}

func highlight(node *html.Node, char string) string {
	t := strings.Replace(childText(node), " ", char, -1)
	return fmt.Sprintf("%s%s%s", char, t, char)
}

type selector func(*html.Node) bool

func isTag(a atom.Atom) selector {
	return func(n *html.Node) bool {
		return n.DataAtom == a
	}
}

func hasClass(name string) selector {
	return func(n *html.Node) bool {
		for _, a := range n.Attr {
			if a.Key == "class" {
				for _, c := range strings.Fields(a.Val) {
					if c == name {
						return true
					}
				}
			}
		}
		return false
	}
}

func hasStyle(s Style) selector {
	return func(n *html.Node) bool {
		for rule, s2 := range cssRules {
			if s2 != s {
				continue
			}
			if strings.HasPrefix(rule, ".") && hasClass(rule[1:])(n) {
				return true
			}
			if n.DataAtom.String() == rule {
				return true
			}
		}
		return false
	}
}

func hasAttr(key, val string) selector {
	return func(n *html.Node) bool {
		for _, a := range n.Attr {
			if a.Key == key && a.Val == val {
				return true
			}
		}
		return false
	}
}

func attr(node *html.Node, key string) (value string) {
	for _, attr := range node.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func findAll(node *html.Node, fn selector) (nodes []*html.Node) {
	walk(node, func(n *html.Node) bool {
		if fn(n) {
			nodes = append(nodes, n)
		}
		return true
	})
	return
}

func find(n *html.Node, fn selector) *html.Node {
	var result *html.Node
	walk(n, func(n *html.Node) bool {
		if result != nil {
			return false
		}
		if fn(n) {
			result = n
			return false
		}
		return true
	})
	return result
}

func walk(n *html.Node, fn selector) {
	if fn(n) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c, fn)
		}
	}
}
<commit_msg>html2article: Ignore empty anchor elements and obtain the original url from Google Docs<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This program takes an HTML file and outputs a corresponding article file in
// present format. See: code.google.com/p/go.tools/present
package main

import (
	"bufio"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"

	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
)

func main() {
	flag.Parse()

	err := convert(os.Stdout, os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
}

func convert(w io.Writer, r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}

	style := find(root, isTag(atom.Style))
	parseStyles(style)

	body := find(root, isTag(atom.Body))
	if body == nil {
		return errors.New("couldn't find body")
	}
	article := limitNewlineRuns(makeHeadings(strings.TrimSpace(text(body))))
	_, err = fmt.Fprintf(w, "Title\n\n%s", article)
	return err
}

type Style string

const (
	Bold   Style = "*"
	Italic Style = "_"
	Code   Style = "`"
)

var cssRules = make(map[string]Style)

func parseStyles(style *html.Node) {
	if style == nil || style.FirstChild == nil {
		log.Println("couldn't find styles")
		return
	}
	s := bufio.NewScanner(strings.NewReader(style.FirstChild.Data))

	findRule := func(b []byte, atEOF bool) (advance int, token []byte, err error) {
		if i := bytes.Index(b, []byte("{")); i >= 0 {
			token = bytes.TrimSpace(b[:i])
			advance = i
		}
		return
	}
	findBody := func(b []byte, atEOF bool) (advance int, token []byte, err error) {
		if len(b) == 0 {
			return
		}
		if b[0] != '{' {
			err = fmt.Errorf("expected {, got %c", b[0])
			return
		}
		if i := bytes.Index(b, []byte("}")); i < 0 {
			err = fmt.Errorf("can't find closing }")
			return
		} else {
			token = b[1:i]
			advance = i + 1
		}
		return
	}

	s.Split(findRule)
	for s.Scan() {
		rule := s.Text()
		s.Split(findBody)
		if !s.Scan() {
			break
		}
		b := strings.ToLower(s.Text())
		switch {
		case strings.Contains(b, "italic"):
			cssRules[rule] = Italic
		case strings.Contains(b, "bold"):
			cssRules[rule] = Bold
		case strings.Contains(b, "Consolas") || strings.Contains(b, "Courier New"):
			cssRules[rule] = Code
		}
		s.Split(findRule)
	}
	if err := s.Err(); err != nil {
		log.Println(err)
	}
}

var newlineRun = regexp.MustCompile(`\n\n+`)

func limitNewlineRuns(s string) string {
	return newlineRun.ReplaceAllString(s, "\n\n")
}

func makeHeadings(body string) string {
	buf := new(bytes.Buffer)
	lines := strings.Split(body, "\n")
	for i, s := range lines {
		if i == 0 && !isBoldTitle(s) {
			buf.WriteString("* Introduction\n\n")
		}
		if isBoldTitle(s) {
			s = strings.TrimSpace(strings.Replace(s, "*", " ", -1))
			s = "* " + s
		}
		buf.WriteString(s)
		buf.WriteByte('\n')
	}
	return buf.String()
}

func isBoldTitle(s string) bool {
	return !strings.Contains(s, " ") &&
		strings.HasPrefix(s, "*") &&
		strings.HasSuffix(s, "*")
}

func indent(buf *bytes.Buffer, s string) {
	for _, l := range strings.Split(s, "\n") {
		if l != "" {
			buf.WriteByte('\t')
			buf.WriteString(l)
		}
		buf.WriteByte('\n')
	}
}

func unwrap(buf *bytes.Buffer, s string) {
	var cont bool
	for _, l := range strings.Split(s, "\n") {
		l = strings.TrimSpace(l)
		if len(l) == 0 {
			if cont {
				buf.WriteByte('\n')
				buf.WriteByte('\n')
			}
			cont = false
		} else {
			if cont {
				buf.WriteByte(' ')
			}
			buf.WriteString(l)
			cont = true
		}
	}
}

func text(n *html.Node) string {
	var buf bytes.Buffer
	walk(n, func(n *html.Node) bool {
		switch n.Type {
		case html.TextNode:
			buf.WriteString(n.Data)
			return false
		case html.ElementNode:
			// no-op
		default:
			return true
		}
		a := n.DataAtom
		if a == atom.Span {
			switch {
			case hasStyle(Code)(n):
				a = atom.Code
			case hasStyle(Bold)(n):
				a = atom.B
			case hasStyle(Italic)(n):
				a = atom.I
			}
		}
		switch a {
		case atom.Br:
			buf.WriteByte('\n')
		case atom.P:
			unwrap(&buf, childText(n))
			buf.WriteString("\n\n")
		case atom.Li:
			buf.WriteString("- ")
			unwrap(&buf, childText(n))
			buf.WriteByte('\n')
		case atom.Pre:
			indent(&buf, childText(n))
			buf.WriteByte('\n')
		case atom.A:
			href, text := attr(n, "href"), childText(n)
			// Skip links with no text.
			if strings.TrimSpace(text) == "" {
				break
			}
			// Don't emit empty links.
			if strings.TrimSpace(href) == "" {
				buf.WriteString(text)
				break
			}
			// Use original url for Google Docs redirections.
			if u, err := url.Parse(href); err != nil {
				log.Println("parsing url %q: %v", href, err)
			} else if u.Host == "www.google.com" && u.Path == "/url" {
				href = u.Query().Get("q")
			}
			fmt.Fprintf(&buf, "[[%s][%s]]", href, text)
		case atom.Code:
			buf.WriteString(highlight(n, "`"))
		case atom.B:
			buf.WriteString(highlight(n, "*"))
		case atom.I:
			buf.WriteString(highlight(n, "_"))
		case atom.Img:
			src := attr(n, "src")
			fmt.Fprintf(&buf, ".image %s\n", src)
		case atom.Iframe:
			src, w, h := attr(n, "src"), attr(n, "width"), attr(n, "height")
			fmt.Fprintf(&buf, "\n.iframe %s %s %s\n", src, h, w)
		case atom.Param:
			if attr(n, "name") == "movie" {
				// Old style YouTube embed.
				u := attr(n, "value")
				u = strings.Replace(u, "/v/", "/embed/", 1)
				if i := strings.Index(u, "&"); i >= 0 {
					u = u[:i]
				}
				fmt.Fprintf(&buf, "\n.iframe %s 540 304\n", u)
			}
		case atom.Title:
		default:
			return true
		}
		return false
	})
	return buf.String()
}

func childText(node *html.Node) string {
	var buf bytes.Buffer
	for n := node.FirstChild; n != nil; n = n.NextSibling {
		fmt.Fprint(&buf, text(n))
	}
	return buf.String()
}

func highlight(node *html.Node, char string) string {
	t := strings.Replace(childText(node), " ", char, -1)
	return fmt.Sprintf("%s%s%s", char, t, char)
}

type selector func(*html.Node) bool

func isTag(a atom.Atom) selector {
	return func(n *html.Node) bool {
		return n.DataAtom == a
	}
}

func hasClass(name string) selector {
	return func(n *html.Node) bool {
		for _, a := range n.Attr {
			if a.Key == "class" {
				for _, c := range strings.Fields(a.Val) {
					if c == name {
						return true
					}
				}
			}
		}
		return false
	}
}

func hasStyle(s Style) selector {
	return func(n *html.Node) bool {
		for rule, s2 := range cssRules {
			if s2 != s {
				continue
			}
			if strings.HasPrefix(rule, ".") && hasClass(rule[1:])(n) {
				return true
			}
			if n.DataAtom.String() == rule {
				return true
			}
		}
		return false
	}
}

func hasAttr(key, val string) selector {
	return func(n *html.Node) bool {
		for _, a := range n.Attr {
			if a.Key == key && a.Val == val {
				return true
			}
		}
		return false
	}
}

func attr(node *html.Node, key string) (value string) {
	for _, attr := range node.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func findAll(node *html.Node, fn selector) (nodes []*html.Node) {
	walk(node, func(n *html.Node) bool {
		if fn(n) {
			nodes = append(nodes, n)
		}
		return true
	})
	return
}

func find(n *html.Node, fn selector) *html.Node {
	var result *html.Node
	walk(n, func(n *html.Node) bool {
		if result != nil {
			return false
		}
		if fn(n) {
			result = n
			return false
		}
		return true
	})
	return result
}

func walk(n *html.Node, fn selector) {
	if fn(n) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c, fn)
		}
	}
}
<|endoftext|>


<commit_before>package realtime

import (
	"errors"
	"fmt"
	"socialapi/models"
	"socialapi/request"
)

type channelUpdatedEventType string

var (
	channelUpdatedEventMessageAddedToChannel     channelUpdatedEventType = "MessageAddedToChannel"
	channelUpdatedEventMessageRemovedFromChannel channelUpdatedEventType = "MessageRemovedFromChannel"
	channelUpdatedEventMessageUpdatedAtChannel   channelUpdatedEventType = "MessageListUpdated"
	channelUpdatedEventReplyAdded                channelUpdatedEventType = "ReplyAdded"
	channelUpdatedEventReplyRemoved              channelUpdatedEventType = "ReplyRemoved"
	channelUpdatedEventChannelParticipantUpdated channelUpdatedEventType = "ParticipantUpdated"
)

type channelUpdatedEvent struct {
	Controller           *Controller                `json:"-"`
	Channel              *models.Channel            `json:"channel"`
	ParentChannelMessage *models.ChannelMessage     `json:"channelMessage"`
	ReplyChannelMessage  *models.ChannelMessage     `json:"-"`
	EventType            channelUpdatedEventType    `json:"event"`
	ChannelParticipant   *models.ChannelParticipant `json:"-"`
	UnreadCount          int                        `json:"unreadCount"`
}

// sendChannelUpdatedEvent sends channel updated events
func (cue *channelUpdatedEvent) send() error {
	cue.Controller.log.Debug("sending channel update event %+v", cue)

	if err := cue.validateChannelUpdatedEvents(); err != nil {
		cue.Controller.log.Error(err.Error())
		// this is not an error actually
		return nil
	}

	// fetch all participants of related channel
	// if you ask why we are not sending those messaages to the channel's channel
	// instead of sending events as notifications?, because we are also sending
	// unread counts of the related channel's messages by the notifiee
	participants, err := cue.Channel.FetchParticipantIds(
		// make sure exempt users are getting reatime notifications
		&request.Query{ShowExempt: true},
	)

	if err != nil {
		cue.Controller.log.Error("Error occured while fetching participants %s", err.Error())
		return err
	}

	// if
	if len(participants) == 0 {
		cue.Controller.log.Notice("This channel (%d) doesnt have any participant but we are trying to send an event to it, please investigate", cue.Channel.Id)
		return nil
	}

	for _, accountId := range participants {
		if !cue.isEligibleForBroadcasting(accountId) {
			cue.Controller.log.Debug("not sending event to the creator of this operation %s", cue.EventType)
			continue
		}

		cp := models.NewChannelParticipant()
		cp.ChannelId = cue.Channel.Id
		cp.AccountId = accountId
		if err := cp.FetchParticipant(); err != nil {
			cue.Controller.log.Error("Err: %s, skipping account %d", err.Error(), accountId)
			return nil
		}
		cue.ChannelParticipant = cp

		err := cue.sendForParticipant()
		if err != nil {
			return err
		}

	}

	return nil
}

func (cue *channelUpdatedEvent) isEligibleForBroadcasting(accountId int64) bool {
	// if parent message is empty do send
	// realtime  updates to the client
	if cue.ParentChannelMessage == nil {
		return true
	}

	// if we are gonna send this notification to topic channel
	// do not send to initiator
	if cue.Channel.TypeConstant == models.Channel_TYPE_TOPIC {
		if cue.ParentChannelMessage.AccountId == accountId {
			return false
		}
	}

	// if reply is not set do send this event
	if cue.ReplyChannelMessage == nil {
		return true
	}

	// if parent message's crateor is account
	// dont send it
	// this has introduced some bugs to system, like if someone
	// comments to my post(i also pinned it)
	// i wasnt getting any notification
	// if cue.ParentChannelMessage.AccountId == accountId {
	// 	return false
	// }

	// if reply message's crateor is account
	// dont send it
	if cue.ReplyChannelMessage.AccountId == accountId {
		return false
	}

	return true
}

func (cue *channelUpdatedEvent) validateChannelUpdatedEvents() error {
	// channel shouldnt be nil
	if cue.Channel == nil {
		return fmt.Errorf("Channel is nil")
	}

	// channel id should be set inorder to send event to the channel
	if cue.Channel.Id == 0 {
		return fmt.Errorf("Channel id is not set")
	}

	// filter group events
	// do not send any -updated- event to group channels
	if cue.Channel.TypeConstant == models.Channel_TYPE_GROUP {
		return fmt.Errorf("Not sending group (%s) event", cue.Channel.GroupName)
	}

	// do not send comment events to topic channels
	// other than topic channel, channels persist their messages as replies
	if cue.Channel.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}

	// if we dont have a parent message it means this is a post addition/creation
	if cue.ParentChannelMessage == nil {
		return nil
	}

	// send only post operations the the client
	if cue.ParentChannelMessage.TypeConstant != models.ChannelMessage_TYPE_POST {
		return fmt.Errorf("Not sending non-post (%s) event to topic channel",
			cue.ParentChannelMessage.TypeConstant,
		)
	}

	return nil
}

func (cue *channelUpdatedEvent) sendForParticipant() error {
	if cue.ChannelParticipant == nil {
		return errors.New("Channel Participant is nil")
	}

	count, err := cue.calculateUnreadItemCount()
	if err != nil {
		cue.Controller.log.Notice("Error happened, setting unread count to 0 %s", err.Error())
		count = 0
	}

	cue.UnreadCount = count

	err = cue.Controller.sendNotification(
		cue.ChannelParticipant.AccountId,
		cue.Channel.GroupName,
		ChannelUpdateEventName,
		cue,
	)
	if err != nil {
		cue.Controller.log.Error(err.Error())
	}

	return nil
}

func (cue *channelUpdatedEvent) calculateUnreadItemCount() (int, error) {
	if cue.ParentChannelMessage == nil {
		return models.NewChannelMessageList().UnreadCount(cue.ChannelParticipant)
	}

	// for topic channel unread count will be calculated from unread post count
	if cue.Channel.TypeConstant == models.Channel_TYPE_TOPIC {
		return models.NewChannelMessageList().UnreadCount(cue.ChannelParticipant)
	}

	// from this point we need parent message

	cml, err := cue.Channel.FetchMessageList(cue.ParentChannelMessage.Id)
	if err != nil {
		return 0, err
	}

	isTroll := cml.MetaBits.Is(models.Troll)

	// for pinned posts calculate unread count from message's added at into that channel
	if cue.Channel.TypeConstant == models.Channel_TYPE_PINNED_ACTIVITY {
		// for pinned posts we are calculating unread count from reviseddAt of the
		// regarding channel message list, since only participant for the channel
		// is the owner and we cant use channel_participant for unread counts
		// on the other hand messages should have their own unread count
		// we are specialcasing the pinned posts here
		return models.NewMessageReply().UnreadCount(cml.MessageId, cml.RevisedAt, isTroll)
	}

	// for private messages calculate the unread reply count
	if cue.Channel.TypeConstant == models.Channel_TYPE_PRIVATE_MESSAGE {
		count, err := models.NewMessageReply().UnreadCount(cue.ParentChannelMessage.Id, cue.ChannelParticipant.LastSeenAt, isTroll)
		if err != nil {
			return 0, err
		}

		// if unread count is 0
		// set it to 1 for now
		// because we want to show a notification with a sign
		if count == 0 {
			count = 1
		}

		return count, nil
	}

	cue.Controller.log.Critical("Calculating unread count shouldnt fall here")
	return 0, nil
}
<commit_msg>Social: add doc for specialcasing<commit_after>package realtime

import (
	"errors"
	"fmt"
	"socialapi/models"
	"socialapi/request"
)

type channelUpdatedEventType string

var (
	channelUpdatedEventMessageAddedToChannel     channelUpdatedEventType = "MessageAddedToChannel"
	channelUpdatedEventMessageRemovedFromChannel channelUpdatedEventType = "MessageRemovedFromChannel"
	channelUpdatedEventMessageUpdatedAtChannel   channelUpdatedEventType = "MessageListUpdated"
	channelUpdatedEventReplyAdded                channelUpdatedEventType = "ReplyAdded"
	channelUpdatedEventReplyRemoved              channelUpdatedEventType = "ReplyRemoved"
	channelUpdatedEventChannelParticipantUpdated channelUpdatedEventType = "ParticipantUpdated"
)

type channelUpdatedEvent struct {
	Controller           *Controller                `json:"-"`
	Channel              *models.Channel            `json:"channel"`
	ParentChannelMessage *models.ChannelMessage     `json:"channelMessage"`
	ReplyChannelMessage  *models.ChannelMessage     `json:"-"`
	EventType            channelUpdatedEventType    `json:"event"`
	ChannelParticipant   *models.ChannelParticipant `json:"-"`
	UnreadCount          int                        `json:"unreadCount"`
}

// sendChannelUpdatedEvent sends channel updated events
func (cue *channelUpdatedEvent) send() error {
	cue.Controller.log.Debug("sending channel update event %+v", cue)

	if err := cue.validateChannelUpdatedEvents(); err != nil {
		cue.Controller.log.Error(err.Error())
		// this is not an error actually
		return nil
	}

	// fetch all participants of related channel
	// if you ask why we are not sending those messaages to the channel's channel
	// instead of sending events as notifications?, because we are also sending
	// unread counts of the related channel's messages by the notifiee
	participants, err := cue.Channel.FetchParticipantIds(
		// make sure exempt users are getting reatime notifications
		&request.Query{ShowExempt: true},
	)

	if err != nil {
		cue.Controller.log.Error("Error occured while fetching participants %s", err.Error())
		return err
	}

	// if
	if len(participants) == 0 {
		cue.Controller.log.Notice("This channel (%d) doesnt have any participant but we are trying to send an event to it, please investigate", cue.Channel.Id)
		return nil
	}

	for _, accountId := range participants {
		if !cue.isEligibleForBroadcasting(accountId) {
			cue.Controller.log.Debug("not sending event to the creator of this operation %s", cue.EventType)
			continue
		}

		cp := models.NewChannelParticipant()
		cp.ChannelId = cue.Channel.Id
		cp.AccountId = accountId
		if err := cp.FetchParticipant(); err != nil {
			cue.Controller.log.Error("Err: %s, skipping account %d", err.Error(), accountId)
			return nil
		}
		cue.ChannelParticipant = cp

		err := cue.sendForParticipant()
		if err != nil {
			return err
		}

	}

	return nil
}

func (cue *channelUpdatedEvent) isEligibleForBroadcasting(accountId int64) bool {
	// if parent message is empty do send
	// realtime  updates to the client
	if cue.ParentChannelMessage == nil {
		return true
	}

	// if we are gonna send this notification to topic channel
	// do not send to initiator
	if cue.Channel.TypeConstant == models.Channel_TYPE_TOPIC {
		if cue.ParentChannelMessage.AccountId == accountId {
			return false
		}
	}

	// if reply is not set do send this event
	if cue.ReplyChannelMessage == nil {
		return true
	}

	// if parent message's crateor is account
	// dont send it
	// this has introduced some bugs to system, like if someone
	// comments to my post(i also pinned it)
	// i wasnt getting any notification
	// if cue.ParentChannelMessage.AccountId == accountId {
	// 	return false
	// }

	// if reply message's crateor is account
	// dont send it
	if cue.ReplyChannelMessage.AccountId == accountId {
		return false
	}

	return true
}

func (cue *channelUpdatedEvent) validateChannelUpdatedEvents() error {
	// channel shouldnt be nil
	if cue.Channel == nil {
		return fmt.Errorf("Channel is nil")
	}

	// channel id should be set inorder to send event to the channel
	if cue.Channel.Id == 0 {
		return fmt.Errorf("Channel id is not set")
	}

	// filter group events
	// do not send any -updated- event to group channels
	if cue.Channel.TypeConstant == models.Channel_TYPE_GROUP {
		return fmt.Errorf("Not sending group (%s) event", cue.Channel.GroupName)
	}

	// do not send comment events to topic channels
	// other than topic channel, channels persist their messages as replies
	if cue.Channel.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}

	// if we dont have a parent message it means this is a post addition/creation
	if cue.ParentChannelMessage == nil {
		return nil
	}

	// send only post operations the the client
	if cue.ParentChannelMessage.TypeConstant != models.ChannelMessage_TYPE_POST {
		return fmt.Errorf("Not sending non-post (%s) event to topic channel",
			cue.ParentChannelMessage.TypeConstant,
		)
	}

	return nil
}

func (cue *channelUpdatedEvent) sendForParticipant() error {
	if cue.ChannelParticipant == nil {
		return errors.New("Channel Participant is nil")
	}

	count, err := cue.calculateUnreadItemCount()
	if err != nil {
		cue.Controller.log.Notice("Error happened, setting unread count to 0 %s", err.Error())
		count = 0
	}

	cue.UnreadCount = count

	err = cue.Controller.sendNotification(
		cue.ChannelParticipant.AccountId,
		cue.Channel.GroupName,
		ChannelUpdateEventName,
		cue,
	)
	if err != nil {
		cue.Controller.log.Error(err.Error())
	}

	return nil
}

func (cue *channelUpdatedEvent) calculateUnreadItemCount() (int, error) {
	if cue.ParentChannelMessage == nil {
		return models.NewChannelMessageList().UnreadCount(cue.ChannelParticipant)
	}

	// Topic channels have the normal structure, one channel, many messages,
	// many participants. For topic channel unread count will be calculated from
	// unread post count whithin a channel, base timestamp here is perisisted in
	// ChannelParticipant table as LastSeenAt timestamp. If one message is
	// edited by another user with a new tag, this message will not be marked as
	// read, because we are not looking to createdAt of the channel message
	// list, we are taking AddedAt into consideration here
	if cue.Channel.TypeConstant == models.Channel_TYPE_TOPIC {
		return models.NewChannelMessageList().UnreadCount(cue.ChannelParticipant)
	}

	// from this point we need parent message

	cml, err := cue.Channel.FetchMessageList(cue.ParentChannelMessage.Id)
	if err != nil {
		return 0, err
	}

	isTroll := cml.MetaBits.Is(models.Troll)

	// for pinned posts calculate unread count from message's added at into that channel
	if cue.Channel.TypeConstant == models.Channel_TYPE_PINNED_ACTIVITY {
		// for pinned posts we are calculating unread count from reviseddAt of the
		// regarding channel message list, since only participant for the channel
		// is the owner and we cant use channel_participant for unread counts
		// on the other hand messages should have their own unread count
		// we are specialcasing the pinned posts here
		return models.NewMessageReply().UnreadCount(cml.MessageId, cml.RevisedAt, isTroll)
	}

	// for private messages calculate the unread reply count
	if cue.Channel.TypeConstant == models.Channel_TYPE_PRIVATE_MESSAGE {
		count, err := models.NewMessageReply().UnreadCount(cue.ParentChannelMessage.Id, cue.ChannelParticipant.LastSeenAt, isTroll)
		if err != nil {
			return 0, err
		}

		// if unread count is 0
		// set it to 1 for now
		// because we want to show a notification with a sign
		if count == 0 {
			count = 1
		}

		return count, nil
	}

	cue.Controller.log.Critical("Calculating unread count shouldnt fall here")
	return 0, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "dmlivewiki"
	app.Usage = "dmlivewiki helper"
	app.Author = `Qais "qaisjp" Patankar`
	app.Email = "me@qaisjp.com"
	app.Version = "1.0.6"

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "force, f",
			Usage: "skip confirmation",
		},
		cli.BoolFlag{
			Name:  "delete",
			Usage: "instead of creating files, delete files",
		},
		cli.BoolFlag{
			Name:  "single, s",
			Usage: "parse the directory given, not the subdirectories",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:   "checksum",
			Usage:  "perform a checksum of directories",
			Action: performChecksum,
		},
		{
			Name:   "generate",
			Usage:  "generate dirname.txt Infofile's for the passed directory",
			Action: generateInformation,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "tour",
					Usage: "required: the tour name for this directory",
				},
				cli.StringFlag{
					Name:  "tour-file",
					Usage: "file with list of tracks with alternate vocals",
				},
			},
		},
		{
			Name:   "wiki",
			Usage:  "generate dirname.wiki Wikifile's for the passed directory",
			Action: generateWikifiles,
		},
	}

	app.Run(os.Args)
}

var informationTemplate = `{{.Artist}}
{{.Date}}
{{.Album}}
{{.Tour}}

Lineage:

Notes:

This source is considered Source 1 for this date:
https://www.depechemode-live.com/wiki/{{wikiescape .Date}}_{{wikiescape .Album}}/Source_1

Track list:

{{range .Tracks}}{{.Prefix}}{{printf "%02d" .Index}}. [{{.Duration}}] {{.Title}}{{if .HasAlternateLeadVocalist}} (*){{end}}
{{end}}Total time: {{.Duration}}

Torrent downloaded from https://www.depechemode-live.com`

var wikiTemplate = `== Notes ==

{{.Notes}}

== Listen ==

You can listen to this entire recording below.

<html5media>https://media.depechemode-live.com/stream/{{.FolderName}}/complete.m4a</html5media>

== Track list ==

{{range .Tracks}}#[{{.Duration}}] <sm2>https://media.depechemode-live.com/stream/{{.FolderName}}/{{printf "%02d" .Index}}.m4a</sm2> [[{{.Name}}]]{{if .HasAlternateLeadVocalist}} (*){{end}}
{{end}}*Total time: {{.Duration}}

== Lineage ==

{{.Lineage}}

== Download ==

*[https://depechemode-live.com/torrents/{{.FolderName}}.torrent Download via torrent] - FLAC {{.BPS}}-bit {{.SampleRate}} - {{.Size}}

[[Category:Audience recordings]]
[[Category:Source]]
[[Category:Streamable]]
`

var wikiRegex = `(?:.|[\r\n])+[\r\n]+Lineage: ((?:.|[\r\n]+)*)[\r\n]+Notes: ((?:.|[\r\n]+)*)[\r\n]+This source is considered(?:.|[\r\n]+)*Track list:[\r\n]+[\r\n]+((?:.|[\r\n]+)*)[\r\n]+Total time: (.*)`
<commit_msg>Add www to wiki link<commit_after>package main

import (
	"github.com/codegangsta/cli"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "dmlivewiki"
	app.Usage = "dmlivewiki helper"
	app.Author = `Qais "qaisjp" Patankar`
	app.Email = "me@qaisjp.com"
	app.Version = "1.0.6"

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "force, f",
			Usage: "skip confirmation",
		},
		cli.BoolFlag{
			Name:  "delete",
			Usage: "instead of creating files, delete files",
		},
		cli.BoolFlag{
			Name:  "single, s",
			Usage: "parse the directory given, not the subdirectories",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:   "checksum",
			Usage:  "perform a checksum of directories",
			Action: performChecksum,
		},
		{
			Name:   "generate",
			Usage:  "generate dirname.txt Infofile's for the passed directory",
			Action: generateInformation,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "tour",
					Usage: "required: the tour name for this directory",
				},
				cli.StringFlag{
					Name:  "tour-file",
					Usage: "file with list of tracks with alternate vocals",
				},
			},
		},
		{
			Name:   "wiki",
			Usage:  "generate dirname.wiki Wikifile's for the passed directory",
			Action: generateWikifiles,
		},
	}

	app.Run(os.Args)
}

var informationTemplate = `{{.Artist}}
{{.Date}}
{{.Album}}
{{.Tour}}

Lineage:

Notes:

This source is considered Source 1 for this date:
https://www.depechemode-live.com/wiki/{{wikiescape .Date}}_{{wikiescape .Album}}/Source_1

Track list:

{{range .Tracks}}{{.Prefix}}{{printf "%02d" .Index}}. [{{.Duration}}] {{.Title}}{{if .HasAlternateLeadVocalist}} (*){{end}}
{{end}}Total time: {{.Duration}}

Torrent downloaded from https://www.depechemode-live.com`

var wikiTemplate = `== Notes ==

{{.Notes}}

== Listen ==

You can listen to this entire recording below.

<html5media>https://media.depechemode-live.com/stream/{{.FolderName}}/complete.m4a</html5media>

== Track list ==

{{range .Tracks}}#[{{.Duration}}] <sm2>https://media.depechemode-live.com/stream/{{.FolderName}}/{{printf "%02d" .Index}}.m4a</sm2> [[{{.Name}}]]{{if .HasAlternateLeadVocalist}} (*){{end}}
{{end}}*Total time: {{.Duration}}

== Lineage ==

{{.Lineage}}

== Download ==

*[https://www.depechemode-live.com/torrents/{{.FolderName}}.torrent Download via torrent] - FLAC {{.BPS}}-bit {{.SampleRate}} - {{.Size}}

[[Category:Audience recordings]]
[[Category:Source]]
[[Category:Streamable]]
`

var wikiRegex = `(?:.|[\r\n])+[\r\n]+Lineage: ((?:.|[\r\n]+)*)[\r\n]+Notes: ((?:.|[\r\n]+)*)[\r\n]+This source is considered(?:.|[\r\n]+)*Track list:[\r\n]+[\r\n]+((?:.|[\r\n]+)*)[\r\n]+Total time: (.*)`
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"net"
	"os"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/satori/go.uuid"
	"gopkg.in/olivere/elastic.v3"
)

func main() {
	fmt.Println("starting")

	redisClient, err := redis.Dial("tcp", os.Getenv("REDIS"))
	if err != nil {
		panic(err)
	}
	defer redisClient.Close()

	fmt.Println(os.Getenv("ELASTICSEARCH"))
	elasticsearchClient, err := elastic.NewClient(
		elastic.SetURL("http://" + os.Getenv("ELASTICSEARCH")),
	)
	if err != nil {
		// Handle error
		panic(err)
	}
	exists, err := elasticsearchClient.IndexExists("requestbin").Do()
	if err != nil {
		panic(err)
	}
	if !exists {
		_, err = elasticsearchClient.CreateIndex("requestbin").Do()
		if err != nil {
			// Handle error
			panic(err)
		}
	}

	httpRoot := os.Getenv("ROOT")
	httpPort := os.Getenv("PORT")
	startHTTPServer(httpRoot, httpPort, redisClient, elasticsearchClient)
	startTCPServer(elasticsearchClient)
	fmt.Println("started")

}

func startTCPServer(elasticsearchClient *elastic.Client) {
	fmt.Println("Starting TCP server on port 9999")
	server, err := net.Listen("tcp", ":9999")

	if server == nil {
		panic(fmt.Sprintf("couldn't start listening: %s", err))
	}
	conns := clientConns(server)
	for {
		go handleConn(<-conns, elasticsearchClient)
	}
}

func clientConns(listener net.Listener) chan net.Conn {
	ch := make(chan net.Conn)
	i := 0
	go func() {
		for {
			client, err := listener.Accept()
			if client == nil {
				fmt.Printf(fmt.Sprintf("couldn't accept: %s", err))
				continue
			}
			i++
			fmt.Printf("%d: %v <-> %v\n", i, client.LocalAddr(), client.RemoteAddr())
			client.SetReadDeadline(time.Now().Add(4 * time.Second))
			ch <- client
		}
	}()
	return ch
}

func handleConn(client net.Conn, elasticsearchClient *elastic.Client) {
	b := bufio.NewReader(client)
	var res bytes.Buffer

	buf := make([]byte, 32)
	for {
		size, err := b.Read(buf)
		res.Write(buf[:size])
		if err != nil {
			break
		}
	}
	fmt.Println("read: " + res.String())
	record := struct {
		Content string    `json:"content"`
		Time    time.Time `json:"time"`
	}{
		Content: res.String(),
		Time:    time.Now(),
	}

	_, err := elasticsearchClient.Index().
		Index("requestbin").
		Type("tcp").
		BodyJson(record).
		Id(uuid.NewV4().String()).
		Do()
	if err != nil {
		fmt.Println(err)
	}
	client.Close()
}
<commit_msg>Removed tcp from main<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/garyburd/redigo/redis"
	"gopkg.in/olivere/elastic.v3"
)

func main() {
	fmt.Println("starting")

	redisClient, err := redis.Dial("tcp", os.Getenv("REDIS"))
	if err != nil {
		panic(err)
	}
	defer redisClient.Close()

	fmt.Println(os.Getenv("ELASTICSEARCH"))
	elasticsearchClient, err := elastic.NewClient(
		elastic.SetURL("http://" + os.Getenv("ELASTICSEARCH")),
	)
	if err != nil {
		// Handle error
		panic(err)
	}
	exists, err := elasticsearchClient.IndexExists("requestbin").Do()
	if err != nil {
		panic(err)
	}
	if !exists {
		_, err = elasticsearchClient.CreateIndex("requestbin").Do()
		if err != nil {
			// Handle error
			panic(err)
		}
	}

	httpRoot := os.Getenv("ROOT")
	httpPort := os.Getenv("PORT")

	startHTTPServer(httpRoot, httpPort, redisClient, elasticsearchClient)
	startTCPServer(elasticsearchClient)
	fmt.Println("started")

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"text/template"
	"time"

	"github.com/jcgregorio/piccolo/piccolo"
	"golang.org/x/net/html"
)

const (
	SITE_TITLE = "BitWorking"
	DOMAIN     = "https://bitworking.org/"
	FEED_LEN   = 4
)

var shortMonths = [...]string{
	"Jan",
	"Feb",
	"Mar",
	"Apr",
	"May",
	"Jun",
	"Jul",
	"Aug",
	"Sep",
	"Oct",
	"Nov",
	"Dec",
}

func fatalf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
	os.Exit(1)
}

// ShortMonth returns the short English name of the month ("Jan", "Feb", ...).
func ShortMonth(m time.Month) string { return shortMonths[m-1] }

type datediffer func(time.Time) string

// datediff returns a function that formats the archive entries correctly.
//
// The returned function is a closure that keeps track of the last time.Time it
// saw which it needs to do the formatting correctly.
func datediff() datediffer {
	var last time.Time

	return func(t time.Time) string {
		r := ""
		if t.After(last) {
			r = fmt.Sprintf("foo %#v", t)
		}
		// If years differ, emit year, month, day
		if t.Year() != last.Year() {
			r = fmt.Sprintf("%d
\n
%s
\n
 %d", t.Year(), ShortMonth(t.Month()), t.Day())
		} else if t.Month() != last.Month() {
			r = fmt.Sprintf("%s
\n
 %d", ShortMonth(t.Month()), t.Day())
		} else {
			r = fmt.Sprintf("%d", t.Day())
		}
		last = t
		return r
	}
}

// trunc10 formats a time to just the year, month and day in ISO format.
func trunc10(t time.Time) string {
	return t.Format("2006-01-02")
}

// rfc339 formats a time in RFC3339 format.
func rfc3339(t time.Time) string {
	return t.Format(time.RFC3339)
}

// Templates contains all the parsed templates.
type Templates struct {
	IndexHTML   *template.Template
	IndexAtom   *template.Template
	ArchiveHTML *template.Template
	EntryHTML   *template.Template
}

func loadTemplate(d *piccolo.DocSet, name string) *template.Template {
	funcMap := template.FuncMap{
		"datediff": datediff(),
		"trunc10":  trunc10,
		"rfc3339":  rfc3339,
	}

	fullname := filepath.Join(d.Root, "tpl", name)
	return template.Must(template.New(name).Funcs(funcMap).ParseFiles(fullname))
}

func loadTemplates(d *piccolo.DocSet) *Templates {
	return &Templates{
		IndexHTML:   loadTemplate(d, "index.html"),
		IndexAtom:   loadTemplate(d, "index.atom"),
		ArchiveHTML: loadTemplate(d, "archive.html"),
		EntryHTML:   loadTemplate(d, "entry.html"),
	}
}

// Expand expands the template with the given data.
func Expand(d *piccolo.DocSet, t *template.Template, data interface{}, path string) error {
	dst, err := d.Dest(path)
	if err != nil {
		return err
	}
	dstDir, _ := filepath.Split(dst)
	if err := os.MkdirAll(dstDir, 0755); err != nil {
		return err
	}
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	t.Execute(out, data)
	return nil
}

// SimpleInclude loads the include file given the docset d.
//
func SimpleInclude(d *piccolo.DocSet, filename string) (string, time.Time, error) {
	fullname := filepath.Join(d.Root, filename)

	f, err := os.Open(fullname)
	if err != nil {
		return "", time.Time{}, err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return "", time.Time{}, err
	}
	t := stat.ModTime()
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return "", time.Time{}, err
	}
	return string(b), t, nil
}

// Include loads the include file given the docset d.
//
// Returns the extracted HTML and the time the file was last modified.
func Include(d *piccolo.DocSet, filename, element string) (string, time.Time, error) {
	fullname := filepath.Join(d.Root, "inc", filename)

	f, err := os.Open(fullname)
	if err != nil {
		return "", time.Time{}, err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return "", time.Time{}, err
	}
	t := stat.ModTime()

	doc, err := html.Parse(f)
	if err != nil {
		return "", time.Time{}, err
	}

	var found func(*html.Node)
	children := []*html.Node{}
	found = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == element {
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				children = append(children, c)
			}
			return
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			found(c)
		}
	}
	found(doc)
	return StrFromNodes(children), t, nil
}

// Newest returns the most recent of all the times passed in.
func Newest(times ...time.Time) time.Time {
	newest := times[0]
	for _, t := range times {
		if t.After(newest) {
			newest = t
		}
	}
	return newest
}

// StrFromNodes returns the string of the rendered html.Nodes.
func StrFromNodes(nodes []*html.Node) string {
	buf := bytes.NewBuffer([]byte{})
	for _, h := range nodes {
		html.Render(buf, h)
	}
	return buf.String()
}

// Entry represents a single blog entry.
type Entry struct {
	// Path is the source file path.
	Path string

	// Title is the title of the entry.
	Title string

	// URL is the relative URL of the file.
	URL string

	// Created is the created time.
	Created time.Time

	// Upated is the updated time.
	Updated time.Time

	// Body is the string representation of the body element, w/o
	// the  tags.
	Body string
}

// EntryByCreated is a type that allows sorting Entries by their created time.
type EntryByCreated []*Entry

func (s EntryByCreated) Len() int           { return len(s) }
func (s EntryByCreated) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s EntryByCreated) Less(i, j int) bool { return s[i].Created.After(s[j].Created) }

// TemplateData is the data used for expanding the index and archive (html and atom) templates.
type TemplateData struct {
	// Domain is the domain name the site will be served from.
	Domain string

	SiteTitle string
	Header    string
	InlineCSS string
	Titlebar  string
	Footer    string
	Entries   []*Entry

	// Most recent time anything on the site was updated.
	Updated time.Time
}

func modifiedTime(path string) time.Time {
	mod := time.Time{}
	if stat, err := os.Stat(path); err == nil {
		mod = stat.ModTime()
	}
	return mod
}

func incMust(s string, t time.Time, err error) (string, time.Time) {
	if err != nil {
		log.Fatalf("Error loading header: %v\n", err)
	}
	return s, t
}

func main() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get cwd: %v\n", err)
	}
	d, err := piccolo.NewDocSet(cwd)
	if err != nil {
		log.Fatalf("Error building docset: %v\n", err)
	}
	fmt.Printf("Root: %s\n", d.Root)

	templates := loadTemplates(d)

	headerStr, headerMod := incMust(Include(d, "header.html", "head"))
	inlineCss, inlineCssMod := incMust(SimpleInclude(d, "css/b.css"))
	footerStr, footerMod := incMust(Include(d, "footer.html", "body"))
	titlebarStr, titlebarMod := incMust(Include(d, "titlebar.html", "body"))

	entryMod := modifiedTime(filepath.Join(d.Root, "tpl", "entry.html"))

	incMod := Newest(headerMod, inlineCssMod, footerMod, titlebarMod, entryMod)

	oneentry := make([]*Entry, 1)
	data := &TemplateData{
		Domain:    DOMAIN,
		SiteTitle: SITE_TITLE,
		Header:    headerStr,
		InlineCSS: string(inlineCss),
		Titlebar:  titlebarStr,
		Footer:    footerStr,
		Entries:   oneentry,
	}

	entries := make([]*Entry, 0)

	// Walk the docset and copy over files, possibly transformed.  Collect all
	// the entries along the way.
	walker := func(path string, info os.FileInfo, err error) error {
		attr, err := d.Path(path)
		if err != nil {
			return err
		}
		if info.IsDir() && attr.Has(piccolo.IGNORE) {
			return filepath.SkipDir
		}
		dest, err := d.Dest(path)
		if err != nil {
			return err
		}
		destMod := modifiedTime(dest)
		if !info.IsDir() && attr.Has(piccolo.INCLUDE) {
			if filepath.Ext(path) == ".html" {
				fileinfo, err := piccolo.CreationDateSaved(path)
				if err != nil {
					return err
				}
				if err := piccolo.LaTex(fileinfo.Node, d.Root); err != nil {
					fmt.Printf("Error: expanding LaTex: %s", err)
				}
				url, err := d.URL(path)
				if err != nil {
					return err
				}
				entries = append(entries, &Entry{
					Path:    path,
					Title:   fileinfo.Title,
					URL:     url,
					Created: fileinfo.Created,
					Updated: fileinfo.Updated,
				})
				if Newest(fileinfo.Updated, incMod).After(destMod) {
					fmt.Printf("INCLUDE:  %v\n", dest)

					// Use the data for template expansion, but with only one entry in it.
					data.Entries[0] = entries[len(entries)-1]
					data.Entries[0].Body = StrFromNodes(fileinfo.Body())
					if err := Expand(d, templates.EntryHTML, data, path); err != nil {
						return err
					}
				}
			}
		}
		if !info.IsDir() && attr.Has(piccolo.VERBATIM) {
			if info.ModTime().After(destMod) {
				fmt.Printf("VERBATIM: %v\n", dest)
				if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
					return err
				}
				dst, err := os.Create(dest)
				if err != nil {
					return err
				}
				defer dst.Close()
				src, err := os.Open(path)
				if err != nil {
					return err
				}
				defer src.Close()
				_, err = io.Copy(dst, src)
				if err != nil {
					return err
				}
			}
		}
		return nil
	}
	err = filepath.Walk(d.Root, walker)
	if err != nil {
		fatalf("Error walking: %v\n", err)
	}

	sort.Sort(EntryByCreated(entries))
	data.Entries = entries

	// TODO(jcgregorio) This is actually wrong, need to sort by Updated first, as if anyone cares.
	data.Updated = entries[0].Updated

	if err := Expand(d, templates.ArchiveHTML, data, filepath.Join(d.Archive, "index.html")); err != nil {
		fatalf("Error building archive: %v\n", err)
	}

	// Take the first 10 items from the list, expand the Body, then pass to templates.
	latest := entries[:FEED_LEN]
	for _, e := range latest {
		fi, _ := piccolo.CreationDateSaved(e.Path)
		if err := piccolo.LaTex(fi.Node, d.Root); err != nil {
			fmt.Printf("Error: expanding LaTex: %s", err)
		}
		e.Body = StrFromNodes(fi.Body())
	}
	data.Entries = latest

	if err := Expand(d, templates.IndexHTML, data, filepath.Join(d.Main, "index.html")); err != nil {
		fatalf("Error building archive: %v\n", err)
	}

	if err := Expand(d, templates.IndexAtom, data, filepath.Join(d.Feed, "index.atom")); err != nil {
		fatalf("Error building feed: %v\n", err)
	}
}
<commit_msg>Fix src of inlined CSS<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"text/template"
	"time"

	"github.com/jcgregorio/piccolo/piccolo"
	"golang.org/x/net/html"
)

const (
	SITE_TITLE = "BitWorking"
	DOMAIN     = "https://bitworking.org/"
	FEED_LEN   = 4
)

var shortMonths = [...]string{
	"Jan",
	"Feb",
	"Mar",
	"Apr",
	"May",
	"Jun",
	"Jul",
	"Aug",
	"Sep",
	"Oct",
	"Nov",
	"Dec",
}

func fatalf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
	os.Exit(1)
}

// ShortMonth returns the short English name of the month ("Jan", "Feb", ...).
func ShortMonth(m time.Month) string { return shortMonths[m-1] }

type datediffer func(time.Time) string

// datediff returns a function that formats the archive entries correctly.
//
// The returned function is a closure that keeps track of the last time.Time it
// saw which it needs to do the formatting correctly.
func datediff() datediffer {
	var last time.Time

	return func(t time.Time) string {
		r := ""
		if t.After(last) {
			r = fmt.Sprintf("foo %#v", t)
		}
		// If years differ, emit year, month, day
		if t.Year() != last.Year() {
			r = fmt.Sprintf("%d
\n
%s
\n
 %d", t.Year(), ShortMonth(t.Month()), t.Day())
		} else if t.Month() != last.Month() {
			r = fmt.Sprintf("%s
\n
 %d", ShortMonth(t.Month()), t.Day())
		} else {
			r = fmt.Sprintf("%d", t.Day())
		}
		last = t
		return r
	}
}

// trunc10 formats a time to just the year, month and day in ISO format.
func trunc10(t time.Time) string {
	return t.Format("2006-01-02")
}

// rfc339 formats a time in RFC3339 format.
func rfc3339(t time.Time) string {
	return t.Format(time.RFC3339)
}

// Templates contains all the parsed templates.
type Templates struct {
	IndexHTML   *template.Template
	IndexAtom   *template.Template
	ArchiveHTML *template.Template
	EntryHTML   *template.Template
}

func loadTemplate(d *piccolo.DocSet, name string) *template.Template {
	funcMap := template.FuncMap{
		"datediff": datediff(),
		"trunc10":  trunc10,
		"rfc3339":  rfc3339,
	}

	fullname := filepath.Join(d.Root, "tpl", name)
	return template.Must(template.New(name).Funcs(funcMap).ParseFiles(fullname))
}

func loadTemplates(d *piccolo.DocSet) *Templates {
	return &Templates{
		IndexHTML:   loadTemplate(d, "index.html"),
		IndexAtom:   loadTemplate(d, "index.atom"),
		ArchiveHTML: loadTemplate(d, "archive.html"),
		EntryHTML:   loadTemplate(d, "entry.html"),
	}
}

// Expand expands the template with the given data.
func Expand(d *piccolo.DocSet, t *template.Template, data interface{}, path string) error {
	dst, err := d.Dest(path)
	if err != nil {
		return err
	}
	dstDir, _ := filepath.Split(dst)
	if err := os.MkdirAll(dstDir, 0755); err != nil {
		return err
	}
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	t.Execute(out, data)
	return nil
}

// SimpleInclude loads the include file given the docset d.
//
func SimpleInclude(d *piccolo.DocSet, filename string) (string, time.Time, error) {
	fullname := filepath.Join(d.Root, filename)

	f, err := os.Open(fullname)
	if err != nil {
		return "", time.Time{}, err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return "", time.Time{}, err
	}
	t := stat.ModTime()
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return "", time.Time{}, err
	}
	return string(b), t, nil
}

// Include loads the include file given the docset d.
//
// Returns the extracted HTML and the time the file was last modified.
func Include(d *piccolo.DocSet, filename, element string) (string, time.Time, error) {
	fullname := filepath.Join(d.Root, "inc", filename)

	f, err := os.Open(fullname)
	if err != nil {
		return "", time.Time{}, err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return "", time.Time{}, err
	}
	t := stat.ModTime()

	doc, err := html.Parse(f)
	if err != nil {
		return "", time.Time{}, err
	}

	var found func(*html.Node)
	children := []*html.Node{}
	found = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == element {
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				children = append(children, c)
			}
			return
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			found(c)
		}
	}
	found(doc)
	return StrFromNodes(children), t, nil
}

// Newest returns the most recent of all the times passed in.
func Newest(times ...time.Time) time.Time {
	newest := times[0]
	for _, t := range times {
		if t.After(newest) {
			newest = t
		}
	}
	return newest
}

// StrFromNodes returns the string of the rendered html.Nodes.
func StrFromNodes(nodes []*html.Node) string {
	buf := bytes.NewBuffer([]byte{})
	for _, h := range nodes {
		html.Render(buf, h)
	}
	return buf.String()
}

// Entry represents a single blog entry.
type Entry struct {
	// Path is the source file path.
	Path string

	// Title is the title of the entry.
	Title string

	// URL is the relative URL of the file.
	URL string

	// Created is the created time.
	Created time.Time

	// Upated is the updated time.
	Updated time.Time

	// Body is the string representation of the body element, w/o
	// the  tags.
	Body string
}

// EntryByCreated is a type that allows sorting Entries by their created time.
type EntryByCreated []*Entry

func (s EntryByCreated) Len() int           { return len(s) }
func (s EntryByCreated) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s EntryByCreated) Less(i, j int) bool { return s[i].Created.After(s[j].Created) }

// TemplateData is the data used for expanding the index and archive (html and atom) templates.
type TemplateData struct {
	// Domain is the domain name the site will be served from.
	Domain string

	SiteTitle string
	Header    string
	InlineCSS string
	Titlebar  string
	Footer    string
	Entries   []*Entry

	// Most recent time anything on the site was updated.
	Updated time.Time
}

func modifiedTime(path string) time.Time {
	mod := time.Time{}
	if stat, err := os.Stat(path); err == nil {
		mod = stat.ModTime()
	}
	return mod
}

func incMust(s string, t time.Time, err error) (string, time.Time) {
	if err != nil {
		log.Fatalf("Error loading header: %v\n", err)
	}
	return s, t
}

func main() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get cwd: %v\n", err)
	}
	d, err := piccolo.NewDocSet(cwd)
	if err != nil {
		log.Fatalf("Error building docset: %v\n", err)
	}
	fmt.Printf("Root: %s\n", d.Root)

	templates := loadTemplates(d)

	headerStr, headerMod := incMust(Include(d, "header.html", "head"))
	inlineCss, inlineCssMod := incMust(SimpleInclude(d, "out/prefixed.css"))
	footerStr, footerMod := incMust(Include(d, "footer.html", "body"))
	titlebarStr, titlebarMod := incMust(Include(d, "titlebar.html", "body"))

	entryMod := modifiedTime(filepath.Join(d.Root, "tpl", "entry.html"))

	incMod := Newest(headerMod, inlineCssMod, footerMod, titlebarMod, entryMod)

	oneentry := make([]*Entry, 1)
	data := &TemplateData{
		Domain:    DOMAIN,
		SiteTitle: SITE_TITLE,
		Header:    headerStr,
		InlineCSS: string(inlineCss),
		Titlebar:  titlebarStr,
		Footer:    footerStr,
		Entries:   oneentry,
	}

	entries := make([]*Entry, 0)

	// Walk the docset and copy over files, possibly transformed.  Collect all
	// the entries along the way.
	walker := func(path string, info os.FileInfo, err error) error {
		attr, err := d.Path(path)
		if err != nil {
			return err
		}
		if info.IsDir() && attr.Has(piccolo.IGNORE) {
			return filepath.SkipDir
		}
		dest, err := d.Dest(path)
		if err != nil {
			return err
		}
		destMod := modifiedTime(dest)
		if !info.IsDir() && attr.Has(piccolo.INCLUDE) {
			if filepath.Ext(path) == ".html" {
				fileinfo, err := piccolo.CreationDateSaved(path)
				if err != nil {
					return err
				}
				if err := piccolo.LaTex(fileinfo.Node, d.Root); err != nil {
					fmt.Printf("Error: expanding LaTex: %s", err)
				}
				url, err := d.URL(path)
				if err != nil {
					return err
				}
				entries = append(entries, &Entry{
					Path:    path,
					Title:   fileinfo.Title,
					URL:     url,
					Created: fileinfo.Created,
					Updated: fileinfo.Updated,
				})
				if Newest(fileinfo.Updated, incMod).After(destMod) {
					fmt.Printf("INCLUDE:  %v\n", dest)

					// Use the data for template expansion, but with only one entry in it.
					data.Entries[0] = entries[len(entries)-1]
					data.Entries[0].Body = StrFromNodes(fileinfo.Body())
					if err := Expand(d, templates.EntryHTML, data, path); err != nil {
						return err
					}
				}
			}
		}
		if !info.IsDir() && attr.Has(piccolo.VERBATIM) {
			if info.ModTime().After(destMod) {
				fmt.Printf("VERBATIM: %v\n", dest)
				if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
					return err
				}
				dst, err := os.Create(dest)
				if err != nil {
					return err
				}
				defer dst.Close()
				src, err := os.Open(path)
				if err != nil {
					return err
				}
				defer src.Close()
				_, err = io.Copy(dst, src)
				if err != nil {
					return err
				}
			}
		}
		return nil
	}
	err = filepath.Walk(d.Root, walker)
	if err != nil {
		fatalf("Error walking: %v\n", err)
	}

	sort.Sort(EntryByCreated(entries))
	data.Entries = entries

	// TODO(jcgregorio) This is actually wrong, need to sort by Updated first, as if anyone cares.
	data.Updated = entries[0].Updated

	if err := Expand(d, templates.ArchiveHTML, data, filepath.Join(d.Archive, "index.html")); err != nil {
		fatalf("Error building archive: %v\n", err)
	}

	// Take the first 10 items from the list, expand the Body, then pass to templates.
	latest := entries[:FEED_LEN]
	for _, e := range latest {
		fi, _ := piccolo.CreationDateSaved(e.Path)
		if err := piccolo.LaTex(fi.Node, d.Root); err != nil {
			fmt.Printf("Error: expanding LaTex: %s", err)
		}
		e.Body = StrFromNodes(fi.Body())
	}
	data.Entries = latest

	if err := Expand(d, templates.IndexHTML, data, filepath.Join(d.Main, "index.html")); err != nil {
		fatalf("Error building archive: %v\n", err)
	}

	if err := Expand(d, templates.IndexAtom, data, filepath.Join(d.Feed, "index.atom")); err != nil {
		fatalf("Error building feed: %v\n", err)
	}
}
<|endoftext|>


<commit_before>package proxy

import (
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/mholt/caddy/config/parse"
)

var (
	supportedPolicies map[string]func() Policy = make(map[string]func() Policy)
	proxyHeaders      http.Header              = make(http.Header)
)

type staticUpstream struct {
	from   string
	Hosts  HostPool
	Policy Policy

	FailTimeout time.Duration
	MaxFails    int32
	HealthCheck struct {
		Path     string
		Interval time.Duration
	}
	WithoutPathPrefix string
}

// NewStaticUpstreams parses the configuration input and sets up
// static upstreams for the proxy middleware.
func NewStaticUpstreams(c parse.Dispenser) ([]Upstream, error) {
	var upstreams []Upstream
	for c.Next() {
		upstream := &staticUpstream{
			from:        "",
			Hosts:       nil,
			Policy:      &Random{},
			FailTimeout: 10 * time.Second,
			MaxFails:    1,
		}

		if !c.Args(&upstream.from) {
			return upstreams, c.ArgErr()
		}
		to := c.RemainingArgs()
		if len(to) == 0 {
			return upstreams, c.ArgErr()
		}

		for c.NextBlock() {
			switch c.Val() {
			case "policy":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}

				if policyCreateFunc, ok := supportedPolicies[c.Val()]; ok {
					upstream.Policy = policyCreateFunc()
				} else {
					return upstreams, c.ArgErr()
				}
			case "fail_timeout":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				if dur, err := time.ParseDuration(c.Val()); err == nil {
					upstream.FailTimeout = dur
				} else {
					return upstreams, err
				}
			case "max_fails":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				if n, err := strconv.Atoi(c.Val()); err == nil {
					upstream.MaxFails = int32(n)
				} else {
					return upstreams, err
				}
			case "health_check":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				upstream.HealthCheck.Path = c.Val()
				upstream.HealthCheck.Interval = 30 * time.Second
				if c.NextArg() {
					if dur, err := time.ParseDuration(c.Val()); err == nil {
						upstream.HealthCheck.Interval = dur
					} else {
						return upstreams, err
					}
				}
			case "proxy_header":
				var header, value string
				if !c.Args(&header, &value) {
					return upstreams, c.ArgErr()
				}
				proxyHeaders.Add(header, value)
			case "websocket":
				proxyHeaders.Add("Connection", "{>Connection}")
				proxyHeaders.Add("Upgrade", "{>Upgrade}")
			case "without":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				upstream.WithoutPathPrefix = c.Val()
			}
		}

		upstream.Hosts = make([]*UpstreamHost, len(to))
		for i, host := range to {
			if !strings.HasPrefix(host, "http") {
				host = "http://" + host
			}
			uh := &UpstreamHost{
				Name:         host,
				Conns:        0,
				Fails:        0,
				FailTimeout:  upstream.FailTimeout,
				Unhealthy:    false,
				ExtraHeaders: proxyHeaders,
				CheckDown: func(upstream *staticUpstream) UpstreamHostDownFunc {
					return func(uh *UpstreamHost) bool {
						if uh.Unhealthy {
							return true
						}
						if uh.Fails >= upstream.MaxFails &&
							upstream.MaxFails != 0 {
							return true
						}
						return false
					}
				}(upstream),
				WithoutPathPrefix: upstream.WithoutPathPrefix,
			}
			if baseURL, err := url.Parse(uh.Name); err == nil {
				uh.ReverseProxy = NewSingleHostReverseProxy(baseURL, uh.WithoutPathPrefix)
			} else {
				return upstreams, err
			}
			upstream.Hosts[i] = uh
		}

		if upstream.HealthCheck.Path != "" {
			go upstream.HealthCheckWorker(nil)
		}
		upstreams = append(upstreams, upstream)
	}
	return upstreams, nil
}

// RegisterPolicy adds a custom policy to the proxy.
func RegisterPolicy(name string, policy func() Policy) {
	supportedPolicies[name] = policy
}

func (u *staticUpstream) From() string {
	return u.from
}

func (u *staticUpstream) healthCheck() {
	for _, host := range u.Hosts {
		hostURL := host.Name + u.HealthCheck.Path
		if r, err := http.Get(hostURL); err == nil {
			io.Copy(ioutil.Discard, r.Body)
			r.Body.Close()
			host.Unhealthy = r.StatusCode < 200 || r.StatusCode >= 400
		} else {
			host.Unhealthy = true
		}
	}
}

func (u *staticUpstream) HealthCheckWorker(stop chan struct{}) {
	ticker := time.NewTicker(u.HealthCheck.Interval)
	u.healthCheck()
	for {
		select {
		case <-ticker.C:
			u.healthCheck()
		case <-stop:
			// TODO: the library should provide a stop channel and global
			// waitgroup to allow goroutines started by plugins a chance
			// to clean themselves up.
		}
	}
}

func (u *staticUpstream) Select() *UpstreamHost {
	pool := u.Hosts
	if len(pool) == 1 {
		if pool[0].Down() {
			return nil
		}
		return pool[0]
	}
	allDown := true
	for _, host := range pool {
		if !host.Down() {
			allDown = false
			break
		}
	}
	if allDown {
		return nil
	}

	if u.Policy == nil {
		return (&Random{}).Select(pool)
	}
	return u.Policy.Select(pool)
}
<commit_msg>proxy: Parse error if property is unknown<commit_after>package proxy

import (
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/mholt/caddy/config/parse"
)

var (
	supportedPolicies map[string]func() Policy = make(map[string]func() Policy)
	proxyHeaders      http.Header              = make(http.Header)
)

type staticUpstream struct {
	from   string
	Hosts  HostPool
	Policy Policy

	FailTimeout time.Duration
	MaxFails    int32
	HealthCheck struct {
		Path     string
		Interval time.Duration
	}
	WithoutPathPrefix string
}

// NewStaticUpstreams parses the configuration input and sets up
// static upstreams for the proxy middleware.
func NewStaticUpstreams(c parse.Dispenser) ([]Upstream, error) {
	var upstreams []Upstream
	for c.Next() {
		upstream := &staticUpstream{
			from:        "",
			Hosts:       nil,
			Policy:      &Random{},
			FailTimeout: 10 * time.Second,
			MaxFails:    1,
		}

		if !c.Args(&upstream.from) {
			return upstreams, c.ArgErr()
		}
		to := c.RemainingArgs()
		if len(to) == 0 {
			return upstreams, c.ArgErr()
		}

		for c.NextBlock() {
			switch c.Val() {
			case "policy":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}

				if policyCreateFunc, ok := supportedPolicies[c.Val()]; ok {
					upstream.Policy = policyCreateFunc()
				} else {
					return upstreams, c.ArgErr()
				}
			case "fail_timeout":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				if dur, err := time.ParseDuration(c.Val()); err == nil {
					upstream.FailTimeout = dur
				} else {
					return upstreams, err
				}
			case "max_fails":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				if n, err := strconv.Atoi(c.Val()); err == nil {
					upstream.MaxFails = int32(n)
				} else {
					return upstreams, err
				}
			case "health_check":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				upstream.HealthCheck.Path = c.Val()
				upstream.HealthCheck.Interval = 30 * time.Second
				if c.NextArg() {
					if dur, err := time.ParseDuration(c.Val()); err == nil {
						upstream.HealthCheck.Interval = dur
					} else {
						return upstreams, err
					}
				}
			case "proxy_header":
				var header, value string
				if !c.Args(&header, &value) {
					return upstreams, c.ArgErr()
				}
				proxyHeaders.Add(header, value)
			case "websocket":
				proxyHeaders.Add("Connection", "{>Connection}")
				proxyHeaders.Add("Upgrade", "{>Upgrade}")
			case "without":
				if !c.NextArg() {
					return upstreams, c.ArgErr()
				}
				upstream.WithoutPathPrefix = c.Val()
			default:
				return upstreams, c.Errf("unknown property '%s'", c.Val())
			}
		}

		upstream.Hosts = make([]*UpstreamHost, len(to))
		for i, host := range to {
			if !strings.HasPrefix(host, "http") {
				host = "http://" + host
			}
			uh := &UpstreamHost{
				Name:         host,
				Conns:        0,
				Fails:        0,
				FailTimeout:  upstream.FailTimeout,
				Unhealthy:    false,
				ExtraHeaders: proxyHeaders,
				CheckDown: func(upstream *staticUpstream) UpstreamHostDownFunc {
					return func(uh *UpstreamHost) bool {
						if uh.Unhealthy {
							return true
						}
						if uh.Fails >= upstream.MaxFails &&
							upstream.MaxFails != 0 {
							return true
						}
						return false
					}
				}(upstream),
				WithoutPathPrefix: upstream.WithoutPathPrefix,
			}
			if baseURL, err := url.Parse(uh.Name); err == nil {
				uh.ReverseProxy = NewSingleHostReverseProxy(baseURL, uh.WithoutPathPrefix)
			} else {
				return upstreams, err
			}
			upstream.Hosts[i] = uh
		}

		if upstream.HealthCheck.Path != "" {
			go upstream.HealthCheckWorker(nil)
		}
		upstreams = append(upstreams, upstream)
	}
	return upstreams, nil
}

// RegisterPolicy adds a custom policy to the proxy.
func RegisterPolicy(name string, policy func() Policy) {
	supportedPolicies[name] = policy
}

func (u *staticUpstream) From() string {
	return u.from
}

func (u *staticUpstream) healthCheck() {
	for _, host := range u.Hosts {
		hostURL := host.Name + u.HealthCheck.Path
		if r, err := http.Get(hostURL); err == nil {
			io.Copy(ioutil.Discard, r.Body)
			r.Body.Close()
			host.Unhealthy = r.StatusCode < 200 || r.StatusCode >= 400
		} else {
			host.Unhealthy = true
		}
	}
}

func (u *staticUpstream) HealthCheckWorker(stop chan struct{}) {
	ticker := time.NewTicker(u.HealthCheck.Interval)
	u.healthCheck()
	for {
		select {
		case <-ticker.C:
			u.healthCheck()
		case <-stop:
			// TODO: the library should provide a stop channel and global
			// waitgroup to allow goroutines started by plugins a chance
			// to clean themselves up.
		}
	}
}

func (u *staticUpstream) Select() *UpstreamHost {
	pool := u.Hosts
	if len(pool) == 1 {
		if pool[0].Down() {
			return nil
		}
		return pool[0]
	}
	allDown := true
	for _, host := range pool {
		if !host.Down() {
			allDown = false
			break
		}
	}
	if allDown {
		return nil
	}

	if u.Policy == nil {
		return (&Random{}).Select(pool)
	}
	return u.Policy.Select(pool)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"strings"
	"sync"
)

type database struct {
	AppServer string
	DbServer  string
	DbName    string
	User      string
	Pass      string
}

func main() {
	usr, err := user.Current()
	if err != nil {
		usage("Couldn't obtain the current user err=%v", err)
	}

	databasesFile := usr.HomeDir + "/.databases.json"
	databases := map[string]database{}

	byts, err := ioutil.ReadFile(databasesFile)
	if err != nil {
		usage("Couldn't read [%v] file. err=%v", databasesFile, err)
	}

	err = json.Unmarshal(byts, &databases)
	if err != nil {
		usage("Couldn't unmarshal [%v] file. err=%v", databasesFile, err)
	}

	if len(databases) == 0 {
		usage("Couldn't find any database configurations in [%v] file.", databasesFile)
	}

	sql := readInput(os.Stdin)
	if len(sql) <= 3 {
		usage("No SQL to run. Exiting.")
	}

	if len(os.Args[1:]) == 0 {
		usage("Target database unspecified; where should I run the query?")
	}

	targetDatabases := []string{}
	for _, k := range os.Args[1:] {
		if _, ok := databases[k]; k != "all" && !ok {
			usage("Target database unknown: [%v]", k)
		}
		if k == "all" {
			targetDatabases = nil
			for k := range databases {
				targetDatabases = append(targetDatabases, k)
			}
			break
		}
		targetDatabases = append(targetDatabases, k)
	}

	var wg sync.WaitGroup

	for _, k := range targetDatabases {
		wg.Add(1)
		go func(db database, k string) {
			defer wg.Done()
			runSQL(db, sql, k, len(targetDatabases) > 1)
		}(databases[k], k)
	}

	wg.Wait()
}

func runSQL(db database, sql string, key string, prependKey bool) {
	userOption := ""
	if db.User != "" {
		userOption = fmt.Sprintf("-u %v ", db.User)
	}

	passOption := ""
	if db.Pass != "" {
		passOption = fmt.Sprintf("-p%v ", db.Pass)
	}

	hostOption := ""
	if db.DbServer != "" {
		hostOption = fmt.Sprintf("-h %v ", db.DbServer)
	}

	prepend := ""
	if prependKey {
		prepend = key + "\t"
	}

	mysql := "mysql"
	options := fmt.Sprintf(" -Nsr %v%v%v%v -e ", userOption, passOption, hostOption, db.DbName)

	var cmd *exec.Cmd
	if db.AppServer != "" {
		query := fmt.Sprintf(`'%v'`, strings.Replace(sql, `'`, `'"'"'`, -1))
		cmd = exec.Command("ssh", db.AppServer, mysql+options+query)
	} else {
		args := append(trimEmpty(strings.Split(options, " ")), sql)
		cmd = exec.Command("mysql", args...)
	}

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Printf("Cannot create pipe for STDOUT of running command on %v; not running.\n", key)
		return
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Printf("Cannot create pipe for STDERR of running command on %v; not running.\n", key)
		return
	}

	if err := cmd.Start(); err != nil {
		log.Printf("Cannot start command on %v; not running.\n", key)
		return
	}

	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		fmt.Println(prepend + scanner.Text())
	}

	stderrLines := []string{}
	scanner = bufio.NewScanner(stderr)
	for scanner.Scan() {
		stderrLines = append(stderrLines, scanner.Text())
	}

	if len(stderrLines) > 0 {
		log.Println(key + " had errors:")
		for _, v := range stderrLines {
			log.Println(key + " [ERROR] " + v)
		}
	}

	cmd.Wait()
}

func readInput(r io.Reader) string {
	ls := []string{}
	var err error
	rd := bufio.NewReader(r)

	for {
		var s string
		s, err = rd.ReadString('\n')
		if err == io.EOF {
			return strings.Join(ls, " ")
		}
		s = strings.TrimSpace(s)
		if len(s) == 0 {
			continue
		}
		ls = append(ls, strings.TrimSpace(s))
	}
}

func trimEmpty(s []string) []string {
	var r []string
	for _, str := range s {
		if str != "" {
			r = append(r, str)
		}
	}
	return r
}
<commit_msg>Returns 0 if no errors, 1 otherwise.<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"strings"
	"sync"
)

type database struct {
	AppServer string
	DbServer  string
	DbName    string
	User      string
	Pass      string
}

func main() {
	usr, err := user.Current()
	if err != nil {
		usage("Couldn't obtain the current user err=%v", err)
	}

	databasesFile := usr.HomeDir + "/.databases.json"
	databases := map[string]database{}

	byts, err := ioutil.ReadFile(databasesFile)
	if err != nil {
		usage("Couldn't read [%v] file. err=%v", databasesFile, err)
	}

	err = json.Unmarshal(byts, &databases)
	if err != nil {
		usage("Couldn't unmarshal [%v] file. err=%v", databasesFile, err)
	}

	if len(databases) == 0 {
		usage("Couldn't find any database configurations in [%v] file.", databasesFile)
	}

	sql := readInput(os.Stdin)
	if len(sql) <= 3 {
		usage("No SQL to run. Exiting.")
	}

	if len(os.Args[1:]) == 0 {
		usage("Target database unspecified; where should I run the query?")
	}

	targetDatabases := []string{}
	for _, k := range os.Args[1:] {
		if _, ok := databases[k]; k != "all" && !ok {
			usage("Target database unknown: [%v]", k)
		}
		if k == "all" {
			targetDatabases = nil
			for k := range databases {
				targetDatabases = append(targetDatabases, k)
			}
			break
		}
		targetDatabases = append(targetDatabases, k)
	}

	var wg sync.WaitGroup

	finalResult := true
	for _, k := range targetDatabases {
		wg.Add(1)
		go func(db database, k string) {
			defer wg.Done()
			if r := runSQL(db, sql, k, len(targetDatabases) > 1); !r {
				finalResult = false
			}
		}(databases[k], k)
	}

	wg.Wait()

	if !finalResult {
		os.Exit(1)
	}
	os.Exit(0)
}

func runSQL(db database, sql string, key string, prependKey bool) bool {
	userOption := ""
	if db.User != "" {
		userOption = fmt.Sprintf("-u %v ", db.User)
	}

	passOption := ""
	if db.Pass != "" {
		passOption = fmt.Sprintf("-p%v ", db.Pass)
	}

	hostOption := ""
	if db.DbServer != "" {
		hostOption = fmt.Sprintf("-h %v ", db.DbServer)
	}

	prepend := ""
	if prependKey {
		prepend = key + "\t"
	}

	mysql := "mysql"
	options := fmt.Sprintf(" -Nsr %v%v%v%v -e ", userOption, passOption, hostOption, db.DbName)

	var cmd *exec.Cmd
	if db.AppServer != "" {
		query := fmt.Sprintf(`'%v'`, strings.Replace(sql, `'`, `'"'"'`, -1))
		cmd = exec.Command("ssh", db.AppServer, mysql+options+query)
	} else {
		args := append(trimEmpty(strings.Split(options, " ")), sql)
		cmd = exec.Command("mysql", args...)
	}

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Printf("Cannot create pipe for STDOUT of running command on %v; not running.\n", key)
		return false
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Printf("Cannot create pipe for STDERR of running command on %v; not running.\n", key)
		return false
	}

	if err := cmd.Start(); err != nil {
		log.Printf("Cannot start command on %v; not running.\n", key)
		return false
	}

	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		fmt.Println(prepend + scanner.Text())
	}

	stderrLines := []string{}
	scanner = bufio.NewScanner(stderr)
	for scanner.Scan() {
		stderrLines = append(stderrLines, scanner.Text())
	}

	cmd.Wait()

	result := true
	if len(stderrLines) > 0 {
		result = false
		log.Println(key + " had errors:")
		for _, v := range stderrLines {
			log.Println(key + " [ERROR] " + v)
		}
	}

	return result
}

func readInput(r io.Reader) string {
	ls := []string{}
	var err error
	rd := bufio.NewReader(r)

	for {
		var s string
		s, err = rd.ReadString('\n')
		if err == io.EOF {
			return strings.Join(ls, " ")
		}
		s = strings.TrimSpace(s)
		if len(s) == 0 {
			continue
		}
		ls = append(ls, strings.TrimSpace(s))
	}
}

func trimEmpty(s []string) []string {
	var r []string
	for _, str := range s {
		if str != "" {
			r = append(r, str)
		}
	}
	return r
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package globalconfig

import (
	"encoding/json"
	"io/ioutil"

	log "github.com/Sirupsen/logrus"

	"github.com/codegangsta/cli"
)

type config struct {
	Flags *flagConfig `json:"flags"`
}

// FlagConfig struct has all of the snapd flags
type flagConfig struct {
	LogPath          *string `json:"log-path"`
	LogLevel         *int    `json:"log-level"`
	MaxProcs         *int    `json:"max-procs"`
	DisableAPI       *bool   `json:"disable-api"`
	APIPort          *int    `json:"api-port"`
	AutodiscoverPath *string `json:"auto-discover"`
	MaxRunning       *int    `json:"max-running-plugins"`
	PluginTrust      *int    `json:"plugin-trust"`
	KeyringPaths     *string `json:"keyring-paths"`
	Cachestr         *string `json:"cache-expiration"`
	IsTribeEnabled   *bool   `json:"tribe"`
	TribeSeed        *string `json:"tribe-seed"`
	TribeNodeName    *string `json:"tribe-node-name"`
	TribeAddr        *string `json:"tribe-addr"`
	TribePort        *int    `json:"tribe-port"`
	RestHTTPS        *bool   `json:"rest-https"`
	RestKey          *string `json:"rest-key"`
	RestCert         *string `json:"rest-cert"`
}

// NewConfig returns a reference to a global config type for the snap daemon
func NewConfig() *config {
	return &config{}
}

func (f *config) LoadConfig(path string) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		log.WithFields(log.Fields{
			"block":   "LoadConfig",
			"_module": "flags",
			"error":   err.Error(),
			"path":    path,
		}).Fatal("unable to read config")
	}
	err = json.Unmarshal(b, &f)
	if err != nil {
		log.WithFields(log.Fields{
			"block":   "LoadConfig",
			"_module": "flags",
			"error":   err.Error(),
			"path":    path,
		}).Fatal("invalid config")
	}
}

// GetFlagInt eturns the integer value for the flag to be used by snapd
func GetFlagInt(ctx *cli.Context, cfgVal *int, flag string) int {
	// Checks if the flag is in the config and if the command line flag is not set
	if cfgVal != nil && !ctx.IsSet(flag) {
		return *cfgVal
	}
	return ctx.Int(flag)
}

// GetFlagBool returns the boolean value for the flag to be used by snapd
func GetFlagBool(ctx *cli.Context, cfgVal *bool, flag string) bool {
	// Checks if the flag is in the config and if the command line flag is not set
	if cfgVal != nil && !ctx.IsSet(flag) {
		return *cfgVal
	}
	return ctx.Bool(flag)
}

// GetFlagString returns the string value for the flag to be used by snapd
func GetFlagString(ctx *cli.Context, cfgVal *string, flag string) string {
	// Checks if the flag is in the config and if the command line flag is not set
	if cfgVal != nil && !ctx.IsSet(flag) {
		return *cfgVal
	}
	return ctx.String(flag)
}
<commit_msg>Removing accidental pointer in flags.go...<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package globalconfig

import (
	"encoding/json"
	"io/ioutil"

	log "github.com/Sirupsen/logrus"

	"github.com/codegangsta/cli"
)

type config struct {
	Flags flagConfig `json:"flags"`
}

// FlagConfig struct has all of the snapd flags
type flagConfig struct {
	LogPath          *string `json:"log-path"`
	LogLevel         *int    `json:"log-level"`
	MaxProcs         *int    `json:"max-procs"`
	DisableAPI       *bool   `json:"disable-api"`
	APIPort          *int    `json:"api-port"`
	AutodiscoverPath *string `json:"auto-discover"`
	MaxRunning       *int    `json:"max-running-plugins"`
	PluginTrust      *int    `json:"plugin-trust"`
	KeyringPaths     *string `json:"keyring-paths"`
	Cachestr         *string `json:"cache-expiration"`
	IsTribeEnabled   *bool   `json:"tribe"`
	TribeSeed        *string `json:"tribe-seed"`
	TribeNodeName    *string `json:"tribe-node-name"`
	TribeAddr        *string `json:"tribe-addr"`
	TribePort        *int    `json:"tribe-port"`
	RestHTTPS        *bool   `json:"rest-https"`
	RestKey          *string `json:"rest-key"`
	RestCert         *string `json:"rest-cert"`
}

// NewConfig returns a reference to a global config type for the snap daemon
func NewConfig() *config {
	return &config{}
}

func (f *config) LoadConfig(path string) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		log.WithFields(log.Fields{
			"block":   "LoadConfig",
			"_module": "flags",
			"error":   err.Error(),
			"path":    path,
		}).Fatal("unable to read config")
	}
	err = json.Unmarshal(b, &f)
	if err != nil {
		log.WithFields(log.Fields{
			"block":   "LoadConfig",
			"_module": "flags",
			"error":   err.Error(),
			"path":    path,
		}).Fatal("invalid config")
	}
}

// GetFlagInt eturns the integer value for the flag to be used by snapd
func GetFlagInt(ctx *cli.Context, cfgVal *int, flag string) int {
	// Checks if the flag is in the config and if the command line flag is not set
	if cfgVal != nil && !ctx.IsSet(flag) {
		return *cfgVal
	}
	return ctx.Int(flag)
}

// GetFlagBool returns the boolean value for the flag to be used by snapd
func GetFlagBool(ctx *cli.Context, cfgVal *bool, flag string) bool {
	// Checks if the flag is in the config and if the command line flag is not set
	if cfgVal != nil && !ctx.IsSet(flag) {
		return *cfgVal
	}
	return ctx.Bool(flag)
}

// GetFlagString returns the string value for the flag to be used by snapd
func GetFlagString(ctx *cli.Context, cfgVal *string, flag string) string {
	// Checks if the flag is in the config and if the command line flag is not set
	if cfgVal != nil && !ctx.IsSet(flag) {
		return *cfgVal
	}
	return ctx.String(flag)
}
<|endoftext|>


<commit_before>// Copyright 2019 Cisco Systems, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRATIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// creates snat crs.

package hostagent

import (
	"context"
	nodeInfov1 "github.com/noironetworks/aci-containers/pkg/nodeinfo/apis/aci.snat/v1"
	nodeInfoclientset "github.com/noironetworks/aci-containers/pkg/nodeinfo/clientset/versioned"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"reflect"
)

func (agent *HostAgent) InformNodeInfo(nodeInfoClient *nodeInfoclientset.Clientset, snatpolicies map[string]struct{}) bool {
	if nodeInfoClient == nil {
		agent.log.Debug("nodeInfo or Kube clients are not intialized")
		return true
	}
	nodeInfo, err := nodeInfoClient.AciV1().NodeInfos(agent.config.AciSnatNamespace).Get(context.TODO(), agent.config.NodeName, metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			nodeInfoInstance := &nodeInfov1.NodeInfo{
				ObjectMeta: metav1.ObjectMeta{
					Name:      agent.config.NodeName,
					Namespace: agent.config.AciSnatNamespace,
				},
				Spec: nodeInfov1.NodeInfoSpec{
					SnatPolicyNames: snatpolicies,
					Macaddress:      agent.config.UplinkMacAdress,
				},
			}
			_, err = nodeInfoClient.AciV1().NodeInfos(agent.config.AciSnatNamespace).Create(context.TODO(), nodeInfoInstance, metav1.CreateOptions{})
		}
	} else {
		if !reflect.DeepEqual(nodeInfo.Spec.SnatPolicyNames, snatpolicies) {
			nodeInfo.Spec.SnatPolicyNames = snatpolicies
			_, err = nodeInfoClient.AciV1().NodeInfos(agent.config.AciSnatNamespace).Update(context.TODO(), nodeInfo, metav1.UpdateOptions{})
		}
	}
	if err == nil {
		agent.log.Debug("NodeInfo Update Successful..")
		return true
	}
	return false
}
<commit_msg>After hostagent restart Policy UID is missing in EP file. 1. This issue is seen becuase the nodeinfo CRD is present with snatpolicy and restart the hostagent,    then after recomputing the policies. if the policyinfo  matches with the nodeinfo CRD there is no sync is triggred.    now we are explicitly triggering the Epfile update if info matches to avoid this issue.<commit_after>// Copyright 2019 Cisco Systems, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRATIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// creates snat crs.

package hostagent

import (
	"context"
	nodeInfov1 "github.com/noironetworks/aci-containers/pkg/nodeinfo/apis/aci.snat/v1"
	nodeInfoclientset "github.com/noironetworks/aci-containers/pkg/nodeinfo/clientset/versioned"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"reflect"
)

func (agent *HostAgent) InformNodeInfo(nodeInfoClient *nodeInfoclientset.Clientset, snatpolicies map[string]struct{}) bool {
	if nodeInfoClient == nil {
		agent.log.Debug("nodeInfo or Kube clients are not intialized")
		return true
	}
	nodeInfo, err := nodeInfoClient.AciV1().NodeInfos(agent.config.AciSnatNamespace).Get(context.TODO(), agent.config.NodeName, metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			nodeInfoInstance := &nodeInfov1.NodeInfo{
				ObjectMeta: metav1.ObjectMeta{
					Name:      agent.config.NodeName,
					Namespace: agent.config.AciSnatNamespace,
				},
				Spec: nodeInfov1.NodeInfoSpec{
					SnatPolicyNames: snatpolicies,
					Macaddress:      agent.config.UplinkMacAdress,
				},
			}
			_, err = nodeInfoClient.AciV1().NodeInfos(agent.config.AciSnatNamespace).Create(context.TODO(), nodeInfoInstance, metav1.CreateOptions{})
		}
	} else {
		if !reflect.DeepEqual(nodeInfo.Spec.SnatPolicyNames, snatpolicies) {
			nodeInfo.Spec.SnatPolicyNames = snatpolicies
			_, err = nodeInfoClient.AciV1().NodeInfos(agent.config.AciSnatNamespace).Update(context.TODO(), nodeInfo, metav1.UpdateOptions{})
		} else {
			// This case can hit restart of the Hostagent and having  same number of policeis present in nodinfo crd.
			agent.indexMutex.Lock()
			var poduids []string
			for name := range snatpolicies {
				for uuid, _ := range agent.snatPods[name] {
					poduids = append(poduids, uuid)
				}
			}
			agent.updateEpFiles(poduids)
			agent.indexMutex.Unlock()
		}
	}
	if err == nil {
		agent.log.Debug("NodeInfo Update Successful..")
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

func main() {
}
<commit_msg>Add empty flags<commit_after>package main

import (
	"flag"
)

func main() {
	flag.Parse()
}
<|endoftext|>