<commit_before>/*
Copyright IBM Corp All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"syscall"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric-lib-go/healthz"
	"github.com/hyperledger/fabric/integration/nwo"
	"github.com/hyperledger/fabric/integration/runner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var _ = Describe("Health", func() {
	var (
		testDir string
		client  *docker.Client
		network *nwo.Network
		process ifrit.Process
	)

	BeforeEach(func() {
		var err error
		testDir, err = ioutil.TempDir("", "e2e")
		Expect(err).NotTo(HaveOccurred())

		client, err = docker.NewClientFromEnv()
		Expect(err).NotTo(HaveOccurred())

		config := nwo.BasicKafka()
		config.Consensus.Brokers = 3

		network = nwo.New(config, testDir, client, BasePort(), components)
		network.GenerateConfigTree()
		network.Bootstrap()
	})

	AfterEach(func() {
		if process != nil {
			process.Signal(syscall.SIGTERM)
			Eventually(process.Wait, network.EventuallyTimeout).Should(Receive())
		}
		if network != nil {
			network.Cleanup()
		}
		os.RemoveAll(testDir)
	})

	Context("when the docker config is bad", func() {
		It("fails the health check", func() {
			peer := network.Peer("Org1", "peer0")
			core := network.ReadPeerConfig(peer)
			core.VM.Endpoint = "127.0.0.1:0" // bad endpoint
			network.WritePeerConfig(peer, core)

			peerRunner := network.PeerRunner(peer)
			process = ginkgomon.Invoke(peerRunner)
			Eventually(process.Ready()).Should(BeClosed())

			authClient, _ := PeerOperationalClients(network, peer)
			healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort))
			statusCode, status := DoHealthCheck(authClient, healthURL)
			Expect(statusCode).To(Equal(http.StatusServiceUnavailable))
			Expect(status.Status).To(Equal("Service Unavailable"))
			Expect(status.FailedChecks).To(ConsistOf(
				healthz.FailedCheck{Component: "docker", Reason: "failed to connect to Docker daemon: invalid endpoint"},
			))
		})
	})

	Describe("CouchDB health checks", func() {
		var (
			couchAddr    string
			authClient   *http.Client
			healthURL    string
			peer         *nwo.Peer
			couchProcess ifrit.Process
		)

		BeforeEach(func() {
			couchDB := &runner.CouchDB{}
			couchProcess = ifrit.Invoke(couchDB)
			Eventually(couchProcess.Ready(), runner.DefaultStartTimeout).Should(BeClosed())
			Consistently(couchProcess.Wait()).ShouldNot(Receive())
			couchAddr = couchDB.Address()

			peer = network.Peer("Org1", "peer0")
			core := network.ReadPeerConfig(peer)
			core.Ledger.State.StateDatabase = "CouchDB"
			core.Ledger.State.CouchDBConfig.CouchDBAddress = couchAddr
			network.WritePeerConfig(peer, core)

			peerRunner := network.PeerRunner(peer)
			process = ginkgomon.Invoke(peerRunner)
			Eventually(process.Ready()).Should(BeClosed())

			authClient, _ = PeerOperationalClients(network, peer)
			healthURL = fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort))
		})

		AfterEach(func() {
			couchProcess.Signal(syscall.SIGTERM)
			Eventually(couchProcess.Wait(), network.EventuallyTimeout).Should(Receive())
		})

		Context("when CouchDB is configured and available", func() {
			It("passes the health check when CouchDB is listening", func() {
				statusCode, status := DoHealthCheck(authClient, healthURL)
				Expect(statusCode).To(Equal(http.StatusOK))
				Expect(status.Status).To(Equal("OK"))
			})
		})

		PContext("when CouchDB is unavailable", func() {
			BeforeEach(func() {
				couchProcess.Signal(syscall.SIGTERM)
				Eventually(couchProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			})

			It("fails the health check", func() {
				statusCode, status := DoHealthCheck(authClient, healthURL)
				Expect(statusCode).To(Equal(http.StatusServiceUnavailable))
				Expect(status.Status).To(Equal("Service Unavailable"))
				Expect(status.FailedChecks[0].Component).To(Equal("couchdb"))
				Expect(status.FailedChecks[0].Reason).Should((HavePrefix(fmt.Sprintf("failed to connect to couch db [Head http://%s: dial tcp %s: ", couchAddr, couchAddr))))
			})
		})
	})

	Describe("Kafka health checks", func() {
		var (
			oProcess ifrit.Process
			zProcess ifrit.Process
			kProcess []ifrit.Process

			authClient *http.Client
			healthURL  string
		)

		BeforeEach(func() {
			// Start Zookeeper
			zookeepers := []string{}
			zk := network.ZooKeeperRunner(0)
			zookeepers = append(zookeepers, fmt.Sprintf("%s:2181", zk.Name))
			zProcess = ifrit.Invoke(zk)
			Eventually(zProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())

			// Start Kafka Brokers
			for i := 0; i < network.Consensus.Brokers; i++ {
				kafkaRunner := network.BrokerRunner(i, zookeepers)
				kp := ifrit.Invoke(kafkaRunner)
				Eventually(kp.Ready(), network.EventuallyTimeout).Should(BeClosed())
				kProcess = append(kProcess, kp)
			}

			// Start Orderer
			ordererRunner := network.OrdererGroupRunner()
			oProcess = ifrit.Invoke(ordererRunner)
			Eventually(oProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())

			orderer := network.Orderers[0]
			authClient, _ = OrdererOperationalClients(network, orderer)
			healthURL = fmt.Sprintf("https://127.0.0.1:%d/healthz", network.OrdererPort(orderer, nwo.OperationsPort))
		})

		AfterEach(func() {
			if zProcess != nil {
				zProcess.Signal(syscall.SIGTERM)
				Eventually(zProcess.Wait, network.EventuallyTimeout).Should(Receive())
			}
			if oProcess != nil {
				oProcess.Signal(syscall.SIGTERM)
				Eventually(oProcess.Wait, network.EventuallyTimeout).Should(Receive())
			}
			for _, k := range kProcess {
				if k != nil {
					k.Signal(syscall.SIGTERM)
					Eventually(k.Wait, network.EventuallyTimeout).Should(Receive())
				}
			}
		})

		Context("when running health checks on orderer the kafka health check", func() {
			It("returns appropriate response code", func() {
				By("returning 200 when all brokers are online", func() {
					statusCode, status := DoHealthCheck(authClient, healthURL)
					Expect(statusCode).To(Equal(http.StatusOK))
					Expect(status.Status).To(Equal("OK"))
				})

				By("returning a 200 when one of the three brokers goes offline", func() {
					k := kProcess[1]
					k.Signal(syscall.SIGTERM)
					Eventually(k.Wait, network.EventuallyTimeout).Should(Receive())

					var statusCode int
					var status healthz.HealthStatus
					for i := 0; i < 4; i++ {
						statusCode, status = DoHealthCheck(authClient, healthURL)
						if statusCode != http.StatusOK {
							break
						}
						time.Sleep(3 * time.Second)
					}
					Expect(statusCode).To(Equal(http.StatusOK))
					Expect(status.Status).To(Equal("OK"))
				})

				By("returning a 503 when two of the three brokers go offline", func() {
					k := kProcess[0]
					k.Signal(syscall.SIGTERM)
					Eventually(k.Wait, network.EventuallyTimeout).Should(Receive())

					var statusCode int
					var status healthz.HealthStatus
					for i := 0; i < 20; i++ {
						statusCode, status = DoHealthCheck(authClient, healthURL)
						if statusCode != http.StatusOK {
							break
						}
						time.Sleep(3 * time.Second)
					}
					Expect(statusCode).To(Equal(http.StatusServiceUnavailable))
					Expect(status.Status).To(Equal("Service Unavailable"))
					Expect(status.FailedChecks[0].Component).To(Equal("systemchannel/0"))
					Expect(status.FailedChecks[0].Reason).To(MatchRegexp(`\[replica ids: \[\d \d \d\]\]: kafka server: Messages are rejected since there are fewer in-sync replicas than required\.`))
				})
			})
		})
	})
})

func DoHealthCheck(client *http.Client, url string) (int, healthz.HealthStatus) {
	resp, err := client.Get(url)
	Expect(err).NotTo(HaveOccurred())

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	Expect(err).NotTo(HaveOccurred())
	resp.Body.Close()

	var healthStatus healthz.HealthStatus
	err = json.Unmarshal(bodyBytes, &healthStatus)
	Expect(err).NotTo(HaveOccurred())

	return resp.StatusCode, healthStatus
}
<commit_msg>[FAB-14331] - Fix flaky Couch DB health check test<commit_after>/*
Copyright IBM Corp All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"syscall"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric-lib-go/healthz"
	"github.com/hyperledger/fabric/integration/nwo"
	"github.com/hyperledger/fabric/integration/runner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var _ = Describe("Health", func() {
	var (
		testDir string
		client  *docker.Client
		network *nwo.Network
		process ifrit.Process
	)

	BeforeEach(func() {
		var err error
		testDir, err = ioutil.TempDir("", "e2e")
		Expect(err).NotTo(HaveOccurred())

		client, err = docker.NewClientFromEnv()
		Expect(err).NotTo(HaveOccurred())

		config := nwo.BasicKafka()
		config.Consensus.Brokers = 3

		network = nwo.New(config, testDir, client, BasePort(), components)
		network.GenerateConfigTree()
		network.Bootstrap()
	})

	AfterEach(func() {
		if process != nil {
			process.Signal(syscall.SIGTERM)
			Eventually(process.Wait, network.EventuallyTimeout).Should(Receive())
		}
		if network != nil {
			network.Cleanup()
		}
		os.RemoveAll(testDir)
	})

	Context("when the docker config is bad", func() {
		It("fails the health check", func() {
			peer := network.Peer("Org1", "peer0")
			core := network.ReadPeerConfig(peer)
			core.VM.Endpoint = "127.0.0.1:0" // bad endpoint
			network.WritePeerConfig(peer, core)

			peerRunner := network.PeerRunner(peer)
			process = ginkgomon.Invoke(peerRunner)
			Eventually(process.Ready()).Should(BeClosed())

			authClient, _ := PeerOperationalClients(network, peer)
			healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort))
			statusCode, status := DoHealthCheck(authClient, healthURL)
			Expect(statusCode).To(Equal(http.StatusServiceUnavailable))
			Expect(status.Status).To(Equal("Service Unavailable"))
			Expect(status.FailedChecks).To(ConsistOf(
				healthz.FailedCheck{Component: "docker", Reason: "failed to connect to Docker daemon: invalid endpoint"},
			))
		})
	})

	Describe("CouchDB health checks", func() {
		var (
			couchAddr    string
			authClient   *http.Client
			healthURL    string
			peer         *nwo.Peer
			couchProcess ifrit.Process
		)

		BeforeEach(func() {
			couchDB := &runner.CouchDB{}
			couchProcess = ifrit.Invoke(couchDB)
			Eventually(couchProcess.Ready(), runner.DefaultStartTimeout).Should(BeClosed())
			Consistently(couchProcess.Wait()).ShouldNot(Receive())
			couchAddr = couchDB.Address()

			peer = network.Peer("Org1", "peer0")
			core := network.ReadPeerConfig(peer)
			core.Ledger.State.StateDatabase = "CouchDB"
			core.Ledger.State.CouchDBConfig.CouchDBAddress = couchAddr
			network.WritePeerConfig(peer, core)

			peerRunner := network.PeerRunner(peer)
			process = ginkgomon.Invoke(peerRunner)
			Eventually(process.Ready()).Should(BeClosed())

			authClient, _ = PeerOperationalClients(network, peer)
			healthURL = fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort))
		})

		AfterEach(func() {
			couchProcess.Signal(syscall.SIGTERM)
			Eventually(couchProcess.Wait(), network.EventuallyTimeout).Should(Receive())
		})

		When("running health checks on Couch DB", func() {
			It("returns approprate response codes", func() {
				By("returning 200 when able to reach Couch DB", func() {
					statusCode, status := DoHealthCheck(authClient, healthURL)
					Expect(statusCode).To(Equal(http.StatusOK))
					Expect(status.Status).To(Equal("OK"))
				})

				By("returning 503 when unable to reach Couch DB", func() {
					couchProcess.Signal(syscall.SIGTERM)
					Eventually(couchProcess.Wait(), network.EventuallyTimeout).Should(Receive())

					var statusCode int
					var status *healthz.HealthStatus
					Eventually(func() int {
						statusCode, status = DoHealthCheck(authClient, healthURL)
						return statusCode
					}, network.EventuallyTimeout).Should(Equal(http.StatusServiceUnavailable))
					Expect(status.Status).To(Equal("Service Unavailable"))
					Expect(status.FailedChecks[0].Component).To(Equal("couchdb"))
					Expect(status.FailedChecks[0].Reason).Should((HavePrefix(fmt.Sprintf("failed to connect to couch db [Head http://%s: dial tcp %s: ", couchAddr, couchAddr))))
				})
			})
		})
	})

	Describe("Kafka health checks", func() {
		var (
			oProcess ifrit.Process
			zProcess ifrit.Process
			kProcess []ifrit.Process

			authClient *http.Client
			healthURL  string
		)

		BeforeEach(func() {
			// Start Zookeeper
			zookeepers := []string{}
			zk := network.ZooKeeperRunner(0)
			zookeepers = append(zookeepers, fmt.Sprintf("%s:2181", zk.Name))
			zProcess = ifrit.Invoke(zk)
			Eventually(zProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())

			// Start Kafka Brokers
			for i := 0; i < network.Consensus.Brokers; i++ {
				kafkaRunner := network.BrokerRunner(i, zookeepers)
				kp := ifrit.Invoke(kafkaRunner)
				Eventually(kp.Ready(), network.EventuallyTimeout).Should(BeClosed())
				kProcess = append(kProcess, kp)
			}

			// Start Orderer
			ordererRunner := network.OrdererGroupRunner()
			oProcess = ifrit.Invoke(ordererRunner)
			Eventually(oProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())

			orderer := network.Orderers[0]
			authClient, _ = OrdererOperationalClients(network, orderer)
			healthURL = fmt.Sprintf("https://127.0.0.1:%d/healthz", network.OrdererPort(orderer, nwo.OperationsPort))
		})

		AfterEach(func() {
			if zProcess != nil {
				zProcess.Signal(syscall.SIGTERM)
				Eventually(zProcess.Wait, network.EventuallyTimeout).Should(Receive())
			}
			if oProcess != nil {
				oProcess.Signal(syscall.SIGTERM)
				Eventually(oProcess.Wait, network.EventuallyTimeout).Should(Receive())
			}
			for _, k := range kProcess {
				if k != nil {
					k.Signal(syscall.SIGTERM)
					Eventually(k.Wait, network.EventuallyTimeout).Should(Receive())
				}
			}
		})

		Context("when running health checks on orderer the kafka health check", func() {
			It("returns appropriate response code", func() {
				By("returning 200 when all brokers are online", func() {
					statusCode, status := DoHealthCheck(authClient, healthURL)
					Expect(statusCode).To(Equal(http.StatusOK))
					Expect(status.Status).To(Equal("OK"))
				})

				By("returning a 200 when one of the three brokers goes offline", func() {
					k := kProcess[1]
					k.Signal(syscall.SIGTERM)
					Eventually(k.Wait, network.EventuallyTimeout).Should(Receive())

					var statusCode int
					var status *healthz.HealthStatus
					Consistently(func() int {
						statusCode, status = DoHealthCheck(authClient, healthURL)
						return statusCode
					}).Should(Equal(http.StatusOK))
					Expect(status.Status).To(Equal("OK"))
				})

				By("returning a 503 when two of the three brokers go offline", func() {
					k := kProcess[0]
					k.Signal(syscall.SIGTERM)
					Eventually(k.Wait, network.EventuallyTimeout).Should(Receive())

					var statusCode int
					var status *healthz.HealthStatus
					Eventually(func() int {
						statusCode, status = DoHealthCheck(authClient, healthURL)
						return statusCode
					}, network.EventuallyTimeout).Should(Equal(http.StatusServiceUnavailable))
					Expect(status.Status).To(Equal("Service Unavailable"))
					Expect(status.FailedChecks[0].Component).To(Equal("systemchannel/0"))
					Expect(status.FailedChecks[0].Reason).To(MatchRegexp(`\[replica ids: \[\d \d \d\]\]: kafka server: Messages are rejected since there are fewer in-sync replicas than required\.`))
				})
			})
		})
	})
})

func DoHealthCheck(client *http.Client, url string) (int, *healthz.HealthStatus) {
	resp, err := client.Get(url)
	Expect(err).NotTo(HaveOccurred())

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	Expect(err).NotTo(HaveOccurred())
	resp.Body.Close()

	// This occurs when a request to the health check server times out, no body is
	// returned when a timeout occurs
	if len(bodyBytes) == 0 {
		return resp.StatusCode, nil
	}

	healthStatus := &healthz.HealthStatus{}
	err = json.Unmarshal(bodyBytes, healthStatus)
	Expect(err).NotTo(HaveOccurred())

	return resp.StatusCode, healthStatus
}
<|endoftext|>


<commit_before>package fourchan

import "github.com/tiko-chan/bbs"
import "io/ioutil"
import "net/http"
import "fmt"
import "encoding/json"
import "strconv"
import "code.google.com/p/go.net/html"
import "github.com/tiko-chan/goquery"
import "strings"

const threadURL = "http://api.4chan.org/%s/res/%s.json"
const boardURL = "http://api.4chan.org/%s/%d.json"
const catalogURL = "http://api.4chan.org/%s/catalog.json"
const boardListURL = "http://api.4chan.org/boards.js"
const imageURL = "http://images.4chan.org/%s/src/%d.%s"
const thumbnailURL = "http://thumbs.4chan.org/%s/thumb/%d.%s"
const spoilerImageURL = "http://static.4chan.org/image/spoiler.png"

var Hello = bbs.HelloMessage{
	Command:         "hello",
	Name:            "◯chan relay",
	ProtocolVersion: 0,
	Description:     "4chan -> BBS Relay",
	Options:         []string{"imageboard", "readonly", "boards"},
	Access: bbs.AccessInfo{
		// There are no user commands.
		GuestCommands: []string{"hello", "get", "list"},
	},
	Formats:       []string{"html", "text"},
	ServerVersion: "4chan-relay 0.1",
}

type Fourchan struct {
}

// we will only define the stuff we need

type Catalog struct {
	Pages []Page `json:"pages"`
}

type Page struct {
	Number  int             `json:"page"`
	Threads []*FourchanPost `json:"threads"`
}

type Thread struct {
	Posts []*FourchanPost `json:"posts"`
}

type FourchanPost struct {
	Number        int    `json:"no"`
	ReplyTo       int    `json:"resto"`
	Sticky        int    `json:"sticky,omitempty"`
	Closed        int    `json:"closed,omitempty"`
	Date          string `json:"now"`
	Timestamp     int    `json:"time"`
	Name          string `json:"name,omitempty"` //username
	Tripcode      string `json:"trip,omitempty"`
	ID            string `json:"id,omitempty"` //user ID
	Capcode       string `json:"capcode,omitempty"`
	CountryName   string `json:"country_name,omitempty"`
	Email         string `json:"email,omitempty"`
	Subject       string `json:"sub,omitempty"`
	Text          string `json:"com,omitempty"` //HTML
	FileName      int    `json:"tim,omitempty"`
	FileExt       string `json:"ext,omitempty"`
	FileDeleted   int    `json:"filedeleted,omitempty"`
	Spoiler       int    `json:"spoiler,omitempty"`
	OmitedPosts   int    `json:"omitted_posts,omitempty"`
	OmittedImages int    `json:"omitted_images,omitempty"`
	Replies       int    `json:"replies,omitempty"`
	Images        int    `json:"images,omitempty"`
}

func (f *Fourchan) LogIn(m *bbs.LoginCommand) bool {
	return false
}

func (f *Fourchan) LogOut(m *bbs.LogoutCommand) *bbs.OKMessage {
	return nil
}

func (f *Fourchan) IsLoggedIn() bool {
	return false
}

func (f *Fourchan) Get(m *bbs.GetCommand) (tm *bbs.ThreadMessage, em *bbs.ErrorMessage) {
	url := fmt.Sprintf(threadURL, m.Board, m.ThreadID)
	data, code := getBytes(url, false)
	if code == 404 {
		return nil, &bbs.ErrorMessage{"error", "get", fmt.Sprintf("Thread /%s/%s not found.", m.Board, m.ThreadID)}
	} else if code != 200 {
		return nil, &bbs.ErrorMessage{"error", "get", fmt.Sprintf("4chan error %d", code)}
	}

	//4chan json in
	var c = Thread{}
	json.Unmarshal(data, &c)

	if len(c.Posts) == 0 {
		return nil, &bbs.ErrorMessage{"error", "get", "No posts!"}
	}

	//bbs json out
	var messages []*bbs.Message
	op := c.Posts[0]

	for i := range c.Posts {
		t := c.Posts[i]
		text := t.Text
		if m.Format == "text" {
			text = unhtml(text)
		}
		messages = append(messages, &bbs.Message{
			ID:           strconv.Itoa(t.Number),
			Author:       name(t),
			AuthorID:     t.ID,
			Date:         t.Date,
			Text:         text,
			PictureURL:   fmt.Sprintf(imageURL, m.Board, t.FileName, t.FileExt),
			ThumbnailURL: fmt.Sprintf(thumbnailURL, m.Board, t.FileName, t.FileExt),
		})
	}

	return &bbs.ThreadMessage{
		Command:  "msg",
		ID:       strconv.Itoa(op.Number),
		Title:    op.Subject,
		Board:    m.Board,
		Format:   "html",
		Messages: messages,
	}, nil
}

func (f *Fourchan) List(m *bbs.ListCommand) (lm *bbs.ListMessage, em *bbs.ErrorMessage) {
	data, code := getBytes(fmt.Sprintf(catalogURL, m.Query), true)
	if code == 404 {
		return nil, &bbs.ErrorMessage{"error", "list", fmt.Sprintf("Board /%s/ not found.", m.Query)}
	} else if code != 200 {
		return nil, &bbs.ErrorMessage{"error", "list", fmt.Sprintf("4chan error %d", code)}
	}

	var c = Catalog{}
	json.Unmarshal(data, &c)

	var threads []*bbs.ThreadListing

	//turn this into bbs messages
	for page := range c.Pages {
		for i := range c.Pages[page].Threads {
			t := c.Pages[page].Threads[i]

			title := t.Subject
			if title == "" {
				title = summary(html.UnescapeString(t.Text))
			}

			threads = append(threads, &bbs.ThreadListing{
				ID:           strconv.Itoa(t.Number),
				Title:        title,
				Author:       name(t),
				AuthorID:     t.ID,
				Date:         t.Date,
				PostCount:    t.Replies,
				PictureURL:   fmt.Sprintf(imageURL, m.Query, t.FileName, t.FileExt),
				ThumbnailURL: fmt.Sprintf(thumbnailURL, m.Query, t.FileName, t.FileExt),
			})
		}
	}

	lm = &bbs.ListMessage{
		Command: "list",
		Type:    "thread",
		Query:   m.Query,
		Threads: threads,
	}
	return lm, nil
}

func (f *Fourchan) Reply(m *bbs.ReplyCommand) (ok *bbs.OKMessage, em *bbs.ErrorMessage) {
	return nil, &bbs.ErrorMessage{"error", "reply", "This relay is read-only."}
}

func (f *Fourchan) Post(m *bbs.PostCommand) (pm *bbs.OKMessage, em *bbs.ErrorMessage) {
	return nil, &bbs.ErrorMessage{"error", "post", "This relay is read-only."}
}

// takes the first line of a thread's comment for when it has no title
func summary(msg string) string {
	msg = unhtml(msg)
	lines := strings.Split(msg, "\n")
	if len(lines) > 0 {
		msg = lines[0]
	} else {
		return ""
	}

	l := 80
	if len(msg) <= l {
		return msg[0:len(msg)]
	}

	return msg[0:l] + "..."
}

func br2nl(h string) string {
	return strings.Replace(h, "
", "\n", -1)
}

func unhtml(s string) string {
	return stringToDocument(br2nl(s)).Text()
}

func name(t *FourchanPost) string {
	var username string
	if t.Name == "" && t.Tripcode == "" && t.Capcode == "" && t.ID == "" {
		username = "Anonymous"
	} else {
		username = t.Name
	}
	if t.Tripcode != "" {
		if t.Name != "" {
			username = username + " ◆" + t.Tripcode
		} else {
			username = "◆" + t.Tripcode
		}
	}
	if t.Capcode != "" {
		username = username + "!!" + t.Capcode
	}
	if t.ID != "" {
		username = username + " [ID: " + t.ID + "]"
	}
	return username
}

// why moot why
func fudge(b []byte) []byte {
	before := []byte(`{"pages":`)
	after := byte('}')
	return append(append(before, b...), after)
}

func getBytes(url string, shouldFudge bool) (b []byte, statusCode int) {
	resp, _ := http.Get(url)
	b, _ = ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if shouldFudge {
		b = fudge(b)
	}
	return b, resp.StatusCode
}

func stringToDocument(data string) *goquery.Document {
	doc, err := html.Parse(strings.NewReader(data))
	if err != nil {
		panic("HTML Error: stringToDocument()")
	}
	return goquery.NewDocumentFromNode(doc)
}
<commit_msg>fixed thumbnails, spoiler thumbnails<commit_after>package fourchan

import "github.com/tiko-chan/bbs"
import "io/ioutil"
import "net/http"
import "fmt"
import "encoding/json"
import "strconv"
import "code.google.com/p/go.net/html"
import "github.com/tiko-chan/goquery"
import "strings"

const threadURL = "http://api.4chan.org/%s/res/%s.json"
const boardURL = "http://api.4chan.org/%s/%d.json"
const catalogURL = "http://api.4chan.org/%s/catalog.json"
const boardListURL = "http://api.4chan.org/boards.js"
const imageURL = "http://images.4chan.org/%s/src/%d%s"
const thumbnailURL = "http://thumbs.4chan.org/%s/thumb/%d%s"
const spoilerImageURL = "http://static.4chan.org/image/spoiler.png"

var Hello = bbs.HelloMessage{
	Command:         "hello",
	Name:            "◯chan relay",
	ProtocolVersion: 0,
	Description:     "4chan -> BBS Relay",
	Options:         []string{"imageboard", "readonly", "boards"},
	Access: bbs.AccessInfo{
		// There are no user commands.
		GuestCommands: []string{"hello", "get", "list"},
	},
	Formats:       []string{"html", "text"},
	ServerVersion: "4chan-relay 0.1",
}

type Fourchan struct {
}

// we will only define the stuff we need

type Catalog struct {
	Pages []Page `json:"pages"`
}

type Page struct {
	Number  int             `json:"page"`
	Threads []*FourchanPost `json:"threads"`
}

type Thread struct {
	Posts []*FourchanPost `json:"posts"`
}

type FourchanPost struct {
	Number        int    `json:"no"`
	ReplyTo       int    `json:"resto"`
	Sticky        int    `json:"sticky,omitempty"`
	Closed        int    `json:"closed,omitempty"`
	Date          string `json:"now"`
	Timestamp     int    `json:"time"`
	Name          string `json:"name,omitempty"` //username
	Tripcode      string `json:"trip,omitempty"`
	ID            string `json:"id,omitempty"` //user ID
	Capcode       string `json:"capcode,omitempty"`
	CountryName   string `json:"country_name,omitempty"`
	Email         string `json:"email,omitempty"`
	Subject       string `json:"sub,omitempty"`
	Text          string `json:"com,omitempty"` //HTML
	FileTime      uint64 `json:"tim,omitempty"`
	FileExt       string `json:"ext,omitempty"`
	FileDeleted   int    `json:"filedeleted,omitempty"`
	Spoiler       int    `json:"spoiler,omitempty"`
	OmitedPosts   int    `json:"omitted_posts,omitempty"`
	OmittedImages int    `json:"omitted_images,omitempty"`
	Replies       int    `json:"replies,omitempty"`
	Images        int    `json:"images,omitempty"`
}

func (f *Fourchan) LogIn(m *bbs.LoginCommand) bool {
	return false
}

func (f *Fourchan) LogOut(m *bbs.LogoutCommand) *bbs.OKMessage {
	return nil
}

func (f *Fourchan) IsLoggedIn() bool {
	return false
}

func (f *Fourchan) Get(m *bbs.GetCommand) (tm *bbs.ThreadMessage, em *bbs.ErrorMessage) {
	url := fmt.Sprintf(threadURL, m.Board, m.ThreadID)
	data, code := getBytes(url, false)
	if code == 404 {
		return nil, &bbs.ErrorMessage{"error", "get", fmt.Sprintf("Thread /%s/%s not found.", m.Board, m.ThreadID)}
	} else if code != 200 {
		return nil, &bbs.ErrorMessage{"error", "get", fmt.Sprintf("4chan error %d", code)}
	}

	//4chan json in
	var c = Thread{}
	json.Unmarshal(data, &c)

	if len(c.Posts) == 0 {
		return nil, &bbs.ErrorMessage{"error", "get", "No posts!"}
	}

	//bbs json out
	var messages []*bbs.Message
	op := c.Posts[0]

	for i := range c.Posts {
		t := c.Posts[i]
		text := t.Text
		if m.Format == "text" {
			text = unhtml(text)
		}

		thumb := fmt.Sprintf(thumbnailURL, m.Board, t.FileTime, t.FileExt)
		if t.Spoiler != 0 {
			thumb = spoilerImageURL
		}

		if t.FileTime != 0 {
			messages = append(messages, &bbs.Message{
				ID:           strconv.Itoa(t.Number),
				Author:       name(t),
				AuthorID:     t.ID,
				Date:         t.Date,
				Text:         text,
				PictureURL:   fmt.Sprintf(imageURL, m.Board, t.FileTime, t.FileExt),
				ThumbnailURL: thumb,
			})
		} else {
			messages = append(messages, &bbs.Message{
				ID:       strconv.Itoa(t.Number),
				Author:   name(t),
				AuthorID: t.ID,
				Date:     t.Date,
				Text:     text,
			})
		}
	}

	return &bbs.ThreadMessage{
		Command:  "msg",
		ID:       strconv.Itoa(op.Number),
		Title:    op.Subject,
		Board:    m.Board,
		Format:   "html",
		Messages: messages,
	}, nil
}

func (f *Fourchan) List(m *bbs.ListCommand) (lm *bbs.ListMessage, em *bbs.ErrorMessage) {
	data, code := getBytes(fmt.Sprintf(catalogURL, m.Query), true)
	if code == 404 {
		return nil, &bbs.ErrorMessage{"error", "list", fmt.Sprintf("Board /%s/ not found.", m.Query)}
	} else if code != 200 {
		return nil, &bbs.ErrorMessage{"error", "list", fmt.Sprintf("4chan error %d", code)}
	}

	var c = Catalog{}
	json.Unmarshal(data, &c)

	var threads []*bbs.ThreadListing

	//turn this into bbs messages
	for page := range c.Pages {
		for i := range c.Pages[page].Threads {
			t := c.Pages[page].Threads[i]

			title := t.Subject
			if title == "" {
				title = summary(html.UnescapeString(t.Text))
			}

			thumb := fmt.Sprintf(thumbnailURL, m.Query, t.FileTime, t.FileExt)
			if t.Spoiler != 0 {
				thumb = spoilerImageURL
			}

			threads = append(threads, &bbs.ThreadListing{
				ID:           strconv.Itoa(t.Number),
				Title:        title,
				Author:       name(t),
				AuthorID:     t.ID,
				Date:         t.Date,
				PostCount:    t.Replies,
				PictureURL:   fmt.Sprintf(imageURL, m.Query, t.FileTime, t.FileExt),
				ThumbnailURL: thumb,
			})
		}
	}

	lm = &bbs.ListMessage{
		Command: "list",
		Type:    "thread",
		Query:   m.Query,
		Threads: threads,
	}
	return lm, nil
}

func (f *Fourchan) Reply(m *bbs.ReplyCommand) (ok *bbs.OKMessage, em *bbs.ErrorMessage) {
	return nil, &bbs.ErrorMessage{"error", "reply", "This relay is read-only."}
}

func (f *Fourchan) Post(m *bbs.PostCommand) (pm *bbs.OKMessage, em *bbs.ErrorMessage) {
	return nil, &bbs.ErrorMessage{"error", "post", "This relay is read-only."}
}

// takes the first line of a thread's comment for when it has no title
func summary(msg string) string {
	msg = unhtml(msg)
	lines := strings.Split(msg, "\n")
	if len(lines) > 0 {
		msg = lines[0]
	} else {
		return ""
	}

	l := 80
	if len(msg) <= l {
		return msg[0:len(msg)]
	}

	return msg[0:l] + "..."
}

func br2nl(h string) string {
	return strings.Replace(h, "
", "\n", -1)
}

func unhtml(s string) string {
	return stringToDocument(br2nl(s)).Text()
}

func name(t *FourchanPost) string {
	var username string
	if t.Name == "" && t.Tripcode == "" && t.Capcode == "" && t.ID == "" {
		username = "Anonymous"
	} else {
		username = t.Name
	}
	if t.Tripcode != "" {
		if t.Name != "" {
			username = username + " ◆" + t.Tripcode
		} else {
			username = "◆" + t.Tripcode
		}
	}
	if t.Capcode != "" {
		username = username + "!!" + t.Capcode
	}
	if t.ID != "" {
		username = username + " [ID: " + t.ID + "]"
	}
	return username
}

// why moot why
func fudge(b []byte) []byte {
	before := []byte(`{"pages":`)
	after := byte('}')
	return append(append(before, b...), after)
}

func getBytes(url string, shouldFudge bool) (b []byte, statusCode int) {
	resp, _ := http.Get(url)
	b, _ = ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if shouldFudge {
		b = fudge(b)
	}
	return b, resp.StatusCode
}

func stringToDocument(data string) *goquery.Document {
	doc, err := html.Parse(strings.NewReader(data))
	if err != nil {
		panic("HTML Error: stringToDocument()")
	}
	return goquery.NewDocumentFromNode(doc)
}
<|endoftext|>


<commit_before>//
// Blackblog
// Copyright 2012 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"fmt"
	"flag"
	"os"
	"path"
	"sort"
	"strings"
	"template"

	"github.com/russross/blackfriday"
)

var (
	flagSource = flag.String("root", "", "The root directory of all Markdown posts")
	flagPort = flag.String("port", "", "The port to bind to for running the standalone HTTP server")
	flagDest = flag.String("dest", "", "The output directory for running in comiple mode")
)

func main() {
	flag.Parse()

	if *flagSource == "" {
		fmt.Fprintf(os.Stderr, "No -root blog directory specified\n")
		os.Exit(1)
	}

	if *flagPort == "" && *flagDest == "" {
		fmt.Fprintf(os.Stderr, "No -port or -dest flag specified\n")
		os.Exit(2)
	}

	if *flagPort != "" {
		fmt.Fprintf(os.Stderr, "** SERVER NOT IMPLEMENTED **\n")
		os.Exit(-1)
	}

	posts := GetPostsInDirectory(*flagSource)
	postMap, sortList := SortPosts(posts)
	for _, url := range sortList {
		post := postMap[url]
		filePath := path.Join(*flagDest, url)

		source, err := post.GetContents()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading post at %s: %v\n", post.Filename, err)
			continue
		}

		html := RenderPost(post, source)
		makeParentDirIfNecessary(filePath)

		fd, err := os.Create(filePath)
		defer fd.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
			continue
		}
		fd.Write(html)
	}
}

// GetPostsInDirectory recursively examines the directory at the path and finds
// any Markdown (.md) files and returns the corresponding Post objects.
func GetPostsInDirectory(dirPath string) []*Post {
	fd, err := os.Open(dirPath)
	defer fd.Close()
	if err != nil {
		return nil
	}

	files, err := fd.Readdir(-1)
	if err != nil {
		return nil
	}

	var results []*Post
	for _, file := range files {
		filePath := path.Join(dirPath, file.Name)
		if file.IsDirectory() {
			subfiles := GetPostsInDirectory(filePath)
			if subfiles != nil {
				results = append(results, subfiles...)
			}
		} else if strings.HasSuffix(file.Name, ".md") {
			if post, err := NewPostFromPath(filePath); post != nil && err == nil {
				results = append(results, post)
			}
		}
	}

	return results
}

// PostURLMap keys Post objects by their final URL.
type PostURLMap map[string]*Post

// SortPosts creates URLs for each Post and returns a map that links the URL to
// the post and a slice of the URLs in sorted order.
func SortPosts(posts []*Post) (postMap PostURLMap, sorted []string) {
	postMap = make(PostURLMap, len(posts))
	for _, post := range posts {
		url := post.CreateURL()
		postMap[url] = post
		sorted = append(sorted, url)
	}
	sort.Strings(sorted)
	return
}

var kPostHeader = []string{
	"
",
	"
{Title}
",
	"
{Date}
",
	"
",
	"\n",
}

// RenderPost runs the input source through the blackfriday library.
func RenderPost(post *Post, input []byte) []byte {
	tpl := template.New(nil)
	tpl.Parse(strings.Join(kPostHeader, "\n"))

	buf := bytes.NewBuffer([]byte{})
	tpl.Execute(buf, post)

	buf.Write(blackfriday.Markdown(
		input,
		blackfriday.HtmlRenderer(
			blackfriday.HTML_USE_SMARTYPANTS |
				blackfriday.HTML_SMARTYPANTS_LATEX_DASHES,
			"",
			""),
		0))

	return buf.Bytes()
}

func makeParentDirIfNecessary(dir string) {
	parent, _ := path.Split(dir)
	os.MkdirAll(parent, 0755)
}
<commit_msg>Generate an index.html file featuring a list of all the posts.<commit_after>//
// Blackblog
// Copyright 2012 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"fmt"
	"flag"
	"os"
	"path"
	"sort"
	"strings"
	"template"

	"github.com/russross/blackfriday"
)

var (
	flagSource = flag.String("root", "", "The root directory of all Markdown posts")
	flagPort = flag.String("port", "", "The port to bind to for running the standalone HTTP server")
	flagDest = flag.String("dest", "", "The output directory for running in comiple mode")
)

func main() {
	flag.Parse()

	if *flagSource == "" {
		fmt.Fprintf(os.Stderr, "No -root blog directory specified\n")
		os.Exit(1)
	}

	if *flagPort == "" && *flagDest == "" {
		fmt.Fprintf(os.Stderr, "No -port or -dest flag specified\n")
		os.Exit(2)
	}

	if *flagPort != "" {
		fmt.Fprintf(os.Stderr, "** SERVER NOT IMPLEMENTED **\n")
		os.Exit(-1)
	}

	posts := GetPostsInDirectory(*flagSource)
	postMap, sortList := SortPosts(posts)
	for _, url := range sortList {
		post := postMap[url]
		filePath := path.Join(*flagDest, url)

		source, err := post.GetContents()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading post at %s: %v\n", post.Filename, err)
			continue
		}

		html := RenderPost(post, source)
		makeParentDirIfNecessary(filePath)

		fd, err := os.Create(filePath)
		defer fd.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
			continue
		}
		fd.Write(html)
	}

	CreateIndex(path.Join(*flagDest, "index.html"), postMap, sortList)
}

// GetPostsInDirectory recursively examines the directory at the path and finds
// any Markdown (.md) files and returns the corresponding Post objects.
func GetPostsInDirectory(dirPath string) []*Post {
	fd, err := os.Open(dirPath)
	defer fd.Close()
	if err != nil {
		return nil
	}

	files, err := fd.Readdir(-1)
	if err != nil {
		return nil
	}

	var results []*Post
	for _, file := range files {
		filePath := path.Join(dirPath, file.Name)
		if file.IsDirectory() {
			subfiles := GetPostsInDirectory(filePath)
			if subfiles != nil {
				results = append(results, subfiles...)
			}
		} else if strings.HasSuffix(file.Name, ".md") {
			if post, err := NewPostFromPath(filePath); post != nil && err == nil {
				results = append(results, post)
			}
		}
	}

	return results
}

// PostURLMap keys Post objects by their final URL.
type PostURLMap map[string]*Post

// SortPosts creates URLs for each Post and returns a map that links the URL to
// the post and a slice of the URLs in sorted order.
func SortPosts(posts []*Post) (postMap PostURLMap, sorted []string) {
	postMap = make(PostURLMap, len(posts))
	for _, post := range posts {
		url := post.CreateURL()
		postMap[url] = post
		sorted = append(sorted, url)
	}
	sort.Strings(sorted)
	return
}

var kPostHeader = []string{
	"
",
	"
{Title}
",
	"
{Date}
",
	"
",
	"\n",
}

// RenderPost runs the input source through the blackfriday library.
func RenderPost(post *Post, input []byte) []byte {
	tpl := template.New(nil)
	tpl.Parse(strings.Join(kPostHeader, "\n"))

	buf := bytes.NewBuffer([]byte{})
	tpl.Execute(buf, post)

	buf.Write(blackfriday.Markdown(
		input,
		blackfriday.HtmlRenderer(
			blackfriday.HTML_USE_SMARTYPANTS |
				blackfriday.HTML_SMARTYPANTS_LATEX_DASHES,
			"",
			""),
		0))

	return buf.Bytes()
}

func makeParentDirIfNecessary(dir string) {
	parent, _ := path.Split(dir)
	os.MkdirAll(parent, 0755)
}

var kIndex = []string{
	"
Posts
",
	"
",
	"{@|str}",
	"
",
}

var kPost = []string{
	"
",
	"	",
	"		{Date}",
	"		{Title}",
	"	",
	"
",
	"\n",
}

// CreateIndex takes the sorted list of posts and generates HTML output listing
// each one.
func CreateIndex(filepath string, postMap PostURLMap, sortOrder []string) {
	tpl := template.New(nil)
	tpl.Parse(strings.Join(kPost, "\n"))

	buf := bytes.NewBuffer([]byte{})
	for _, url := range sortOrder {
		tpl.Execute(buf, map[string]string{
			"URL": url,
			"Date": postMap[url].Date,
			"Title": postMap[url].Title,
		});
	}

	posts := buf.Bytes()

	buf = bytes.NewBuffer([]byte{})

	tpl = template.New(nil)
	tpl.Parse(strings.Join(kIndex, "\n"))
	tpl.Execute(buf, posts)

	fd, err := os.Create(filepath)
	defer fd.Close()
	if err != nil {
		return
	}

	fd.Write(buf.Bytes())
}
<|endoftext|>


<commit_before>// warren takes home monitoring data and feeds it into
// [influxdb](http://influxdb.org/).
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"syscall"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/huin/gocc"
	ifl "github.com/influxdb/influxdb-go"
)

var (
	configFile = flag.String("config", "", "Path to configuration file")
)

type Config struct {
	LogPath     string
	InfluxDB    ifl.ClientConfig
	CurrentCost CurrentCostConfig
}

type CurrentCostConfig struct {
	Device string
}

func wattsReading(now int64, sensor, id, channel, watts int) []interface{} {
	return []interface{}{now, "cc", sensor, id, channel, watts}
}

func currentCost(cfg *CurrentCostConfig, influxdb *ifl.Client) error {
	msgReader, err := gocc.NewSerialMessageReader(cfg.Device)
	if err != nil {
		return err
	}
	defer msgReader.Close()

	for {
		msg, err := msgReader.ReadMessage()
		if err != nil {
			return err
		}
		now := time.Now().Unix()

		series := []*ifl.Series{
			{
				Name:    "temperature",
				Columns: []string{"time", "source", "value"},
				Points:  [][]interface{}{{now, "cc", msg.Temperature}},
			},
		}

		if msg.Sensor != nil && *msg.Sensor >= 0 && msg.ID != nil {
			pts := [][]interface{}{}

			if msg.Channel1 != nil {
				pts = append(pts, wattsReading(now, *msg.Sensor, *msg.ID, 1, msg.Channel1.Watts))
			}
			if msg.Channel2 != nil {
				pts = append(pts, wattsReading(now, *msg.Sensor, *msg.ID, 2, msg.Channel2.Watts))
			}
			if msg.Channel3 != nil {
				pts = append(pts, wattsReading(now, *msg.Sensor, *msg.ID, 3, msg.Channel3.Watts))
			}

			if len(pts) > 0 {
				series = append(series, &ifl.Series{
					Name:    "watts",
					Columns: []string{"time", "source", "sensor", "id", "channel", "value"},
					Points:  pts,
				})
			}
		}

		if err := influxdb.WriteSeriesWithTimePrecision(series, ifl.Second); err != nil {
			return err
		}
	}
}

func initLogging(logpath string) error {
	if logpath == "" {
		// Leave default logging as STDERR.
		return nil
	}

	f, err := os.OpenFile(logpath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, syscall.S_IWUSR|syscall.S_IRUSR)
	if err != nil {
		return fmt.Errorf("failed to open log file: %v", err)
	}
	log.SetOutput(f)
	return nil
}

func readConfig(filename string) (*Config, error) {
	config := new(Config)
	_, err := toml.DecodeFile(filename, &config)
	if err != nil {
		return nil, err
	}
	return config, nil
}

func main() {
	flag.Parse()
	if *configFile == "" {
		log.Fatal("--config is required with a filename")
	}
	config, err := readConfig(*configFile)

	if err != nil {
		log.Fatal("Failed to read configuration: ", err)
	}
	initLogging(config.LogPath)

	influxdb, err := ifl.NewClient(&config.InfluxDB)
	if err != nil {
		log.Fatal("Failed to connect to influxdb: ", err)
	}

	for {
		err := currentCost(&config.CurrentCost, influxdb)
		log.Print("Current Cost monitoring error: ", err)

		// Avoid tightlooping on recurring failure.
		time.Sleep(5 * time.Second)
	}
}
<commit_msg>Auto-reconnect to influx when encountering errors.<commit_after>// warren takes home monitoring data and feeds it into
// [influxdb](http://influxdb.org/).
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"syscall"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/huin/gocc"
	ifl "github.com/influxdb/influxdb-go"
)

var (
	configFile = flag.String("config", "", "Path to configuration file")
)

type Config struct {
	LogPath     string
	InfluxDB    ifl.ClientConfig
	CurrentCost CurrentCostConfig
}

type CurrentCostConfig struct {
	Device string
}

func wattsReading(now int64, sensor, id, channel, watts int) []interface{} {
	return []interface{}{now, "cc", sensor, id, channel, watts}
}

func currentCost(cfg *CurrentCostConfig, influxChan chan<- []*ifl.Series) error {
	msgReader, err := gocc.NewSerialMessageReader(cfg.Device)
	if err != nil {
		return err
	}
	defer msgReader.Close()

	for {
		msg, err := msgReader.ReadMessage()
		if err != nil {
			return err
		}
		now := time.Now().Unix()

		series := []*ifl.Series{
			{
				Name:    "temperature",
				Columns: []string{"time", "source", "value"},
				Points:  [][]interface{}{{now, "cc", msg.Temperature}},
			},
		}

		if msg.Sensor != nil && *msg.Sensor >= 0 && msg.ID != nil {
			pts := [][]interface{}{}

			if msg.Channel1 != nil {
				pts = append(pts, wattsReading(now, *msg.Sensor, *msg.ID, 1, msg.Channel1.Watts))
			}
			if msg.Channel2 != nil {
				pts = append(pts, wattsReading(now, *msg.Sensor, *msg.ID, 2, msg.Channel2.Watts))
			}
			if msg.Channel3 != nil {
				pts = append(pts, wattsReading(now, *msg.Sensor, *msg.ID, 3, msg.Channel3.Watts))
			}

			if len(pts) > 0 {
				series = append(series, &ifl.Series{
					Name:    "watts",
					Columns: []string{"time", "source", "sensor", "id", "channel", "value"},
					Points:  pts,
				})
			}
		}

		influxChan <- series
	}
}

func influxSender(cfg *ifl.ClientConfig, influxChan <-chan []*ifl.Series) {
	for {
		influxdb, err := ifl.NewClient(cfg)
		if err != nil {
			log.Print("Failed to connect to influxdb: ", err)
		}

		for series := range influxChan {
			if err := influxdb.WriteSeriesWithTimePrecision(series, ifl.Second); err != nil {
				log.Print("Failed to send to influxdb: ", err)
				break
			}
		}

		// Avoid tightlooping on recurring failure.
		time.Sleep(5 * time.Second)
	}
}

func initLogging(logpath string) error {
	if logpath == "" {
		// Leave default logging as STDERR.
		return nil
	}

	f, err := os.OpenFile(logpath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, syscall.S_IWUSR|syscall.S_IRUSR)
	if err != nil {
		return fmt.Errorf("failed to open log file: %v", err)
	}
	log.SetOutput(f)
	return nil
}

func readConfig(filename string) (*Config, error) {
	config := new(Config)
	_, err := toml.DecodeFile(filename, &config)
	if err != nil {
		return nil, err
	}
	return config, nil
}

func main() {
	flag.Parse()
	if *configFile == "" {
		log.Fatal("--config is required with a filename")
	}
	config, err := readConfig(*configFile)

	if err != nil {
		log.Fatal("Failed to read configuration: ", err)
	}
	initLogging(config.LogPath)

	influxChan := make(chan []*ifl.Series)

	go influxSender(&config.InfluxDB, influxChan)

	for {
		err := currentCost(&config.CurrentCost, influxChan)
		log.Print("Current Cost monitoring error: ", err)

		// Avoid tightlooping on recurring failure.
		time.Sleep(5 * time.Second)
	}
}
<|endoftext|>


<commit_before>package dashboard

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"net/http"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"

	"github.com/Cepave/open-falcon-backend/modules/fe/g"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/base"
	"github.com/Cepave/open-falcon-backend/modules/fe/model/dashboard"
	"github.com/Cepave/open-falcon-backend/modules/fe/model/uic"
	"github.com/SlyMarbo/rss"
)

type DashBoardController struct {
	base.BaseController
}

func (this *DashBoardController) EndpRegxqury() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	enp, err := dashboard.QueryEndpintByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(enp) > 0 {
		baseResp.Data["endpoints"] = enp
	} else {
		baseResp.Data["endpoints"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

type xmlEntry struct {
	ID      string `xml:"id"`
	Updated string `xml:"updated"`
}

type xmlData struct {
	EntryList []xmlEntry `xml:"entry"`
}

func (this *DashBoardController) LatestPlugin() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	v := xmlData{}

	c, q_err := dashboard.QueryConfig("atom_addr")
	log.Println("Lastest Plugin atom address value is: ", c.Value)
	if q_err != nil {
		log.Errorln("QueryConfig error: ", q_err)
	} else {
		if resp, err := http.Get(c.Value); err != nil {
			// handle error.
			log.Println("Error retrieving resource:", err)
		} else {
			defer resp.Body.Close()
			xml.NewDecoder(resp.Body).Decode(&v)
		}
	}

	baseResp.Data["EntryList"] = v.EntryList
	this.ServeApiJson(baseResp)
	return
}

//counter query by endpoints
func (this *DashBoardController) CounterQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	endpoints := this.GetString("endpoints", "")
	endpointcheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if endpoints == "" || endpointcheck.MatchString(endpoints) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	endpointsArr := strings.Split(rexstr.ReplaceAllString(endpoints, ""), ",")
	limitNum, _ := this.GetInt("limit", 0)
	metricQuery := this.GetString("metricQuery", "")
	counters, err := dashboard.QueryCounterByEndpoints(endpointsArr, limitNum, metricQuery)
	switch {
	case err != nil:
		this.ResposeError(baseResp, err.Error())
		return
	case len(counters) == 0:
		baseResp.Data["counters"] = []string{}
	default:
		baseResp.Data["counters"] = counters
	}
	this.ServeApiJson(baseResp)
	return
}

//endpoints query by counter
func (this *DashBoardController) EndpointQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	counter := this.GetString("counter", "")
	limitNum, _ := this.GetInt("limit", 0)
	metricQuery := this.GetString("metricQuery", "")
	negateMatch, _ := this.GetBool("negateMatch", false)
	if metricQuery == "" && counter == "" {
		this.ResposeError(baseResp, "query string && query pattern are both empty, please check it")
		return
	}
	endpoints, err := dashboard.QueryEndpointsByCounter(counter, limitNum, metricQuery, negateMatch)
	switch {
	case err != nil:
		this.ResposeError(baseResp, err.Error())
		return
	case len(endpoints) == 0:
		baseResp.Data["endpoints"] = []string{}
	default:
		baseResp.Data["endpoints"] = endpoints
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostGroupQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	hostgroupList, err := dashboard.QueryHostGroupByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hostgroupList) > 0 {
		baseResp.Data["hostgroups"] = hostgroupList
	} else {
		baseResp.Data["hostgroups"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostsQueryByHostGroups() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")
	hosts_resp, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)

	if len(hosts_resp) > 0 {
		baseResp.Data["hosts"] = hosts_resp
	} else {
		baseResp.Data["hosts"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CounterQueryByHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")

	hosts, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hosts) > 0 {
		var endpoints []string
		for _, v := range hosts {
			endpoints = append(endpoints, fmt.Sprintf("\"%v\"", v.Hostname))
		}
		limitNum, _ := this.GetInt("limit", 0)
		metricQuery := this.GetString("metricQuery", "")
		counters, err := dashboard.QueryCounterByEndpoints(endpoints, limitNum, metricQuery)
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		} else if len(counters) > 0 {
			baseResp.Data["counters"] = counters
		} else {
			baseResp.Data["counters"] = []string{}
		}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CountNumOfHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()

	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	} else {
		numberOfteam, err := dashboard.CountNumOfHostGroup()
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		}
		baseResp.Data["count"] = numberOfteam
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRunningPlugin() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	addr := this.GetString("addr", "")
	resp, AgentErr := http.Get(addr)
	baseResp.Data["requestAddr"] = addr
	log.Debugln("response from Agent: ", resp)
	log.Debugln("error message from Agent: ", AgentErr)
	if AgentErr != nil {
		baseResp.Data["errorFromAgent"] = AgentErr.Error()
	} else {
		defer resp.Body.Close()
		data := map[string]interface{}{}
		json.NewDecoder(resp.Body).Decode(&data)
		baseResp.Data["msgFromAgent"] = data["msg"]
		baseResp.Data["dataFromAgent"] = data["data"]
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRegxquryForPlugin() {
	baseResp := this.BasicRespGen()
	session, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	var username *uic.User
	if session.Uid <= 0 {
		baseResp.Data["SessionFlag"] = true
		baseResp.Data["ErrorMsg"] = "Session is not vaild"
	} else {
		baseResp.Data["SessionFlag"] = false
		username = uic.SelectUserById(session.Uid)
		if username.Name != "root" {
			baseResp.Data["SessionFlag"] = true
			baseResp.Data["ErrorMsg"] = "You don't have permission to access this page"
		}
	}
	queryStr := ".+"
	if baseResp.Data["SessionFlag"] == false {
		enp, _ := dashboard.QueryEndpintByNameRegxForOps(queryStr)
		if len(enp) > 0 {
			baseResp.Data["Endpoints"] = enp
		} else {
			baseResp.Data["Endpoints"] = []string{}
		}
	}
	log.Debugln(baseResp)
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRegxquryForOps() {
	this.Data["Shortcut"] = g.Config().Shortcut
	sig := this.Ctx.GetCookie("sig")
	session := uic.ReadSessionBySig(sig)
	var username *uic.User
	if sig == "" || session.Uid <= 0 {
		this.Data["SessionFlag"] = true
		this.Data["ErrorMsg"] = "Session is not vaild"
	} else {
		this.Data["SessionFlag"] = false
		username = uic.SelectUserById(session.Uid)
		if username.Name != "root" {
			this.Data["SessionFlag"] = true
			this.Data["ErrorMsg"] = "You don't have permission to access this page"
		}
	}
	queryStr := this.GetString("queryStr", "")
	this.Data["QueryCondstion"] = queryStr
	if queryStr == "" || this.Data["SessionFlag"] == true {
		this.Data["Init"] = true
	} else {
		enpRow, _ := dashboard.QueryEndpintByNameRegxForOps(queryStr)
		enp := gitInfoAdapter(enpRow)
		if len(enp) > 0 {
			var ips []string
			this.Data["Endpoints"] = enp
			this.Data["Len"] = len(enp)
			for _, en := range enp {
				if en.Ip != "" {
					ips = append(ips, en.Ip)
				}
			}
			this.Data["IP"] = strings.Join(ips, ",")
		} else {
			this.Data["Endpoints"] = []string{}
			this.Data["Len"] = 0
			this.Data["IP"] = ""
		}
	}
	this.TplName = "dashboard/endpoints.html"
}

var commitsInfo []*rss.Item

func gitInfoAdapter(enpRow []dashboard.Hosts) (enp []dashboard.GitInfo) {
	c, q_err := dashboard.QueryConfig("atom_addr")
	log.Println("gitInfoAdapter shows atom address as: ", c.Value)
	if q_err != nil {
		log.Errorln("QueryConfig error: ", q_err)
	}

	feed, err := rss.Fetch(c.Value)
	if err != nil {
		log.Println(err)
	}

	commitsInfo = append(commitsInfo, feed.Items...)
	log.Debugln("commit atom feed is:", feed.Items)
	log.Debugln("commitsInfo is:", commitsInfo)
	for _, host := range enpRow {
		gitInfo := dashboard.GitInfo{Hostname: host.Hostname,
			Ip:            host.Ip,
			AgentVersion:  host.AgentVersion,
			PluginVersion: host.PluginVersion,
			Valid:         false}
		for _, item := range commitsInfo {
			titleArray := strings.Split(item.ID, "/")
			hash := strings.TrimSpace(titleArray[len(titleArray)-1])
			if hash == host.PluginVersion {
				// copy Title and Date column
				gitInfo.Date = item.Date
				gitInfo.Title = item.Title
				gitInfo.Valid = true
				break
			}
		}
		enp = append(enp, gitInfo)
	}

	return
}
<commit_msg>[OWL-480][fe] change debug message<commit_after>package dashboard

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"net/http"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"

	"github.com/Cepave/open-falcon-backend/modules/fe/g"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/base"
	"github.com/Cepave/open-falcon-backend/modules/fe/model/dashboard"
	"github.com/Cepave/open-falcon-backend/modules/fe/model/uic"
	"github.com/SlyMarbo/rss"
)

type DashBoardController struct {
	base.BaseController
}

func (this *DashBoardController) EndpRegxqury() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	enp, err := dashboard.QueryEndpintByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(enp) > 0 {
		baseResp.Data["endpoints"] = enp
	} else {
		baseResp.Data["endpoints"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

type xmlEntry struct {
	ID      string `xml:"id"`
	Updated string `xml:"updated"`
}

type xmlData struct {
	EntryList []xmlEntry `xml:"entry"`
}

func (this *DashBoardController) LatestPlugin() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	v := xmlData{}

	c, q_err := dashboard.QueryConfig("atom_addr")
	if q_err != nil {
		log.Errorln("QueryConfig error: ", q_err)
	} else {
		log.Debugln("Lastest Plugin atom address value is: ", c.Value)
		if resp, err := http.Get(c.Value); err != nil {
			log.Errorln("Error retrieving resource:", err)
		} else {
			defer resp.Body.Close()
			xml.NewDecoder(resp.Body).Decode(&v)
		}
	}

	baseResp.Data["EntryList"] = v.EntryList
	this.ServeApiJson(baseResp)
	return
}

//counter query by endpoints
func (this *DashBoardController) CounterQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	endpoints := this.GetString("endpoints", "")
	endpointcheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if endpoints == "" || endpointcheck.MatchString(endpoints) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	endpointsArr := strings.Split(rexstr.ReplaceAllString(endpoints, ""), ",")
	limitNum, _ := this.GetInt("limit", 0)
	metricQuery := this.GetString("metricQuery", "")
	counters, err := dashboard.QueryCounterByEndpoints(endpointsArr, limitNum, metricQuery)
	switch {
	case err != nil:
		this.ResposeError(baseResp, err.Error())
		return
	case len(counters) == 0:
		baseResp.Data["counters"] = []string{}
	default:
		baseResp.Data["counters"] = counters
	}
	this.ServeApiJson(baseResp)
	return
}

//endpoints query by counter
func (this *DashBoardController) EndpointQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	counter := this.GetString("counter", "")
	limitNum, _ := this.GetInt("limit", 0)
	metricQuery := this.GetString("metricQuery", "")
	negateMatch, _ := this.GetBool("negateMatch", false)
	if metricQuery == "" && counter == "" {
		this.ResposeError(baseResp, "query string && query pattern are both empty, please check it")
		return
	}
	endpoints, err := dashboard.QueryEndpointsByCounter(counter, limitNum, metricQuery, negateMatch)
	switch {
	case err != nil:
		this.ResposeError(baseResp, err.Error())
		return
	case len(endpoints) == 0:
		baseResp.Data["endpoints"] = []string{}
	default:
		baseResp.Data["endpoints"] = endpoints
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostGroupQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	hostgroupList, err := dashboard.QueryHostGroupByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hostgroupList) > 0 {
		baseResp.Data["hostgroups"] = hostgroupList
	} else {
		baseResp.Data["hostgroups"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostsQueryByHostGroups() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")
	hosts_resp, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)

	if len(hosts_resp) > 0 {
		baseResp.Data["hosts"] = hosts_resp
	} else {
		baseResp.Data["hosts"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CounterQueryByHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")

	hosts, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hosts) > 0 {
		var endpoints []string
		for _, v := range hosts {
			endpoints = append(endpoints, fmt.Sprintf("\"%v\"", v.Hostname))
		}
		limitNum, _ := this.GetInt("limit", 0)
		metricQuery := this.GetString("metricQuery", "")
		counters, err := dashboard.QueryCounterByEndpoints(endpoints, limitNum, metricQuery)
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		} else if len(counters) > 0 {
			baseResp.Data["counters"] = counters
		} else {
			baseResp.Data["counters"] = []string{}
		}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CountNumOfHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()

	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	} else {
		numberOfteam, err := dashboard.CountNumOfHostGroup()
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		}
		baseResp.Data["count"] = numberOfteam
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRunningPlugin() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	addr := this.GetString("addr", "")
	resp, AgentErr := http.Get(addr)
	baseResp.Data["requestAddr"] = addr
	log.Debugln("response from Agent: ", resp)
	log.Debugln("error message from Agent: ", AgentErr)
	if AgentErr != nil {
		baseResp.Data["errorFromAgent"] = AgentErr.Error()
	} else {
		defer resp.Body.Close()
		data := map[string]interface{}{}
		json.NewDecoder(resp.Body).Decode(&data)
		baseResp.Data["msgFromAgent"] = data["msg"]
		baseResp.Data["dataFromAgent"] = data["data"]
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRegxquryForPlugin() {
	baseResp := this.BasicRespGen()
	session, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	var username *uic.User
	if session.Uid <= 0 {
		baseResp.Data["SessionFlag"] = true
		baseResp.Data["ErrorMsg"] = "Session is not vaild"
	} else {
		baseResp.Data["SessionFlag"] = false
		username = uic.SelectUserById(session.Uid)
		if username.Name != "root" {
			baseResp.Data["SessionFlag"] = true
			baseResp.Data["ErrorMsg"] = "You don't have permission to access this page"
		}
	}
	queryStr := ".+"
	if baseResp.Data["SessionFlag"] == false {
		enp, _ := dashboard.QueryEndpintByNameRegxForOps(queryStr)
		if len(enp) > 0 {
			baseResp.Data["Endpoints"] = enp
		} else {
			baseResp.Data["Endpoints"] = []string{}
		}
	}
	log.Debugln(baseResp)
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRegxquryForOps() {
	this.Data["Shortcut"] = g.Config().Shortcut
	sig := this.Ctx.GetCookie("sig")
	session := uic.ReadSessionBySig(sig)
	var username *uic.User
	if sig == "" || session.Uid <= 0 {
		this.Data["SessionFlag"] = true
		this.Data["ErrorMsg"] = "Session is not vaild"
	} else {
		this.Data["SessionFlag"] = false
		username = uic.SelectUserById(session.Uid)
		if username.Name != "root" {
			this.Data["SessionFlag"] = true
			this.Data["ErrorMsg"] = "You don't have permission to access this page"
		}
	}
	queryStr := this.GetString("queryStr", "")
	this.Data["QueryCondstion"] = queryStr
	if queryStr == "" || this.Data["SessionFlag"] == true {
		this.Data["Init"] = true
	} else {
		enpRow, _ := dashboard.QueryEndpintByNameRegxForOps(queryStr)
		enp := gitInfoAdapter(enpRow)
		if len(enp) > 0 {
			var ips []string
			this.Data["Endpoints"] = enp
			this.Data["Len"] = len(enp)
			for _, en := range enp {
				if en.Ip != "" {
					ips = append(ips, en.Ip)
				}
			}
			this.Data["IP"] = strings.Join(ips, ",")
		} else {
			this.Data["Endpoints"] = []string{}
			this.Data["Len"] = 0
			this.Data["IP"] = ""
		}
	}
	this.TplName = "dashboard/endpoints.html"
}

var commitsInfo []*rss.Item

func gitInfoAdapter(enpRow []dashboard.Hosts) (enp []dashboard.GitInfo) {
	c, q_err := dashboard.QueryConfig("atom_addr")
	if q_err != nil {
		log.Errorln("QueryConfig error: ", q_err)
	}
	log.Debugln("gitInfoAdapter shows atom address as: ", c.Value)

	feed, err := rss.Fetch(c.Value)
	if err != nil {
		log.Errorln(err)
	}

	commitsInfo = append(commitsInfo, feed.Items...)
	log.Debugln("commit atom feed is:", feed.Items)
	log.Debugln("commitsInfo is:", commitsInfo)
	for _, host := range enpRow {
		gitInfo := dashboard.GitInfo{Hostname: host.Hostname,
			Ip:            host.Ip,
			AgentVersion:  host.AgentVersion,
			PluginVersion: host.PluginVersion,
			Valid:         false}
		for _, item := range commitsInfo {
			titleArray := strings.Split(item.ID, "/")
			hash := strings.TrimSpace(titleArray[len(titleArray)-1])
			if hash == host.PluginVersion {
				// copy Title and Date column
				gitInfo.Date = item.Date
				gitInfo.Title = item.Title
				gitInfo.Valid = true
				break
			}
		}
		enp = append(enp, gitInfo)
	}

	return
}
<|endoftext|>


<commit_before>package steam

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
)

const (
	apiKeyURL         = "https://steamcommunity.com/dev/apikey"
	apiKeyRegisterURL = "https://steamcommunity.com/dev/registerkey"
	apiKeyRevokeURL   = "https://steamcommunity.com/dev/revokekey"

	accessDeniedPattern = "
Access Denied
"
)

var (
	keyRegExp = regexp.MustCompile("
Key: ([0-9A-F]+)
")

	ErrCannotRegisterKey = errors.New("unable to register API key")
	ErrCannotRevokeKey   = errors.New("unable to revoke API key")
	ErrAccessDenied      = errors.New("access is denied")
	ErrKeyNotFound       = errors.New("key not found")
)

func (session *Session) RegisterWebAPIKey(domain string) error {
	resp, err := session.client.PostForm(apiKeyRegisterURL, url.Values{
		"domain":       {domain},
		"agreeToTerms": {"agreed"},
		"sessionid":    {session.sessionID},
		"Submit":       {"Register"},
	})
	if resp != nil {
		resp.Body.Close()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return ErrCannotRegisterKey
	}

	return nil
}

func (session *Session) GetWebAPIKey() (string, error) {
	resp, err := session.client.Get(apiKeyURL)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return "", err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	if m, err := regexp.Match(accessDeniedPattern, body); err != nil {
		return "", err
	} else if m {
		return "", ErrAccessDenied
	}

	submatch := keyRegExp.FindStringSubmatch(string(body))
	if len(submatch) <= 1 {
		return "", ErrKeyNotFound
	}

	session.apiKey = submatch[1]
	return submatch[1], nil
}

func (session *Session) RevokeWebAPIKey() error {
	resp, err := session.client.PostForm(apiKeyRevokeURL, url.Values{
		"revoke":    {"Revoke My Steam Web API Key"},
		"sessionid": {session.sessionID},
	})
	if resp != nil {
		resp.Body.Close()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return ErrCannotRevokeKey
	}

	return nil
}
<commit_msg>webapi: fix revoke<commit_after>package steam

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
)

const (
	apiKeyURL         = "https://steamcommunity.com/dev/apikey"
	apiKeyRegisterURL = "https://steamcommunity.com/dev/registerkey"
	apiKeyRevokeURL   = "https://steamcommunity.com/dev/revokekey"

	accessDeniedPattern = "
Access Denied
"
)

var (
	keyRegExp = regexp.MustCompile("
Key: ([0-9A-F]+)
")

	ErrCannotRegisterKey = errors.New("unable to register API key")
	ErrCannotRevokeKey   = errors.New("unable to revoke API key")
	ErrAccessDenied      = errors.New("access is denied")
	ErrKeyNotFound       = errors.New("key not found")
)

func (session *Session) RegisterWebAPIKey(domain string) error {
	resp, err := session.client.PostForm(apiKeyRegisterURL, url.Values{
		"domain":       {domain},
		"agreeToTerms": {"agreed"},
		"sessionid":    {session.sessionID},
		"Submit":       {"Register"},
	})
	if resp != nil {
		resp.Body.Close()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return ErrCannotRegisterKey
	}

	return nil
}

func (session *Session) GetWebAPIKey() (string, error) {
	resp, err := session.client.Get(apiKeyURL)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return "", err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	if m, err := regexp.Match(accessDeniedPattern, body); err != nil {
		return "", err
	} else if m {
		return "", ErrAccessDenied
	}

	submatch := keyRegExp.FindStringSubmatch(string(body))
	if len(submatch) <= 1 {
		return "", ErrKeyNotFound
	}

	session.apiKey = submatch[1]
	return submatch[1], nil
}

func (session *Session) RevokeWebAPIKey() error {
	resp, err := session.client.PostForm(apiKeyRevokeURL, url.Values{
		"Revoke":    {"Revoke My Steam Web API Key"},
		"sessionid": {session.sessionID},
	})
	if resp != nil {
		resp.Body.Close()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return ErrCannotRevokeKey
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buffer

import (
	"reflect"
	"unsafe"

	"github.com/jacobsa/fuse/internal/fusekernel"
)

// Buffer provides a mechanism for constructing a single contiguous fuse
// message from multiple segments, where the first segment is always a
// fusekernel.OutHeader message.
//
// Must be created with New. Exception: the zero value has Bytes() == nil.
type Buffer struct {
	slice []byte
}

// Create a new buffer whose initial contents are a zeroed fusekernel.OutHeader
// message, and with room enough to grow by extra bytes.
func New(extra uintptr) (b Buffer) {
	const headerSize = unsafe.Sizeof(fusekernel.OutHeader{})
	b.slice = make([]byte, headerSize, headerSize+extra)
	return
}

// Return a pointer to the header at the start of the buffer.
func (b *Buffer) OutHeader() (h *fusekernel.OutHeader) {
	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b.slice))
	h = (*fusekernel.OutHeader)(unsafe.Pointer(sh.Data))
	return
}

// Grow the buffer by the supplied number of bytes, returning a pointer to the
// start of the new segment. The sum of the arguments given to Grow must not
// exceed the argument given to New when creating the buffer.
func (b *Buffer) Grow(size uintptr) (p unsafe.Pointer) {
	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b.slice))
	p = unsafe.Pointer(sh.Data + uintptr(sh.Len))
	b.slice = b.slice[:len(b.slice)+int(size)]
	return
}

// Equivalent to growing by the length of p, then copying p into the new segment.
func (b *Buffer) Append(p []byte) {
	sh := reflect.SliceHeader{
		Data: uintptr(b.Grow(uintptr(len(p)))),
		Len:  len(p),
		Cap:  len(p),
	}

	copy(*(*[]byte)(unsafe.Pointer(&sh)), p)
}

// Equivalent to growing by the length of s, then copying s into the new segment.
func (b *Buffer) AppendString(s string) {
	sh := reflect.SliceHeader{
		Data: uintptr(b.Grow(uintptr(len(s)))),
		Len:  len(s),
		Cap:  len(s),
	}

	copy(*(*[]byte)(unsafe.Pointer(&sh)), s)
}

// Return a reference to the current contents of the buffer.
func (b *Buffer) Bytes() []byte {
	return b.slice
}
<commit_msg>Renamed to OutMessage.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buffer

import (
	"reflect"
	"unsafe"

	"github.com/jacobsa/fuse/internal/fusekernel"
)

// OutMessage provides a mechanism for constructing a single contiguous fuse
// message from multiple segments, where the first segment is always a
// fusekernel.OutHeader message.
//
// Must be created with New. Exception: the zero value has Bytes() == nil.
type OutMessage struct {
	slice []byte
}

// Create a new buffer whose initial contents are a zeroed fusekernel.OutHeader
// message, and with room enough to grow by extra bytes.
func NewOutMessage(extra uintptr) (b OutMessage) {
	const headerSize = unsafe.Sizeof(fusekernel.OutHeader{})
	b.slice = make([]byte, headerSize, headerSize+extra)
	return
}

// Return a pointer to the header at the start of the buffer.
func (b *OutMessage) OutHeader() (h *fusekernel.OutHeader) {
	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b.slice))
	h = (*fusekernel.OutHeader)(unsafe.Pointer(sh.Data))
	return
}

// Grow the buffer by the supplied number of bytes, returning a pointer to the
// start of the new segment. The sum of the arguments given to Grow must not
// exceed the argument given to New when creating the buffer.
func (b *OutMessage) Grow(size uintptr) (p unsafe.Pointer) {
	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b.slice))
	p = unsafe.Pointer(sh.Data + uintptr(sh.Len))
	b.slice = b.slice[:len(b.slice)+int(size)]
	return
}

// Equivalent to growing by the length of p, then copying p into the new segment.
func (b *OutMessage) Append(p []byte) {
	sh := reflect.SliceHeader{
		Data: uintptr(b.Grow(uintptr(len(p)))),
		Len:  len(p),
		Cap:  len(p),
	}

	copy(*(*[]byte)(unsafe.Pointer(&sh)), p)
}

// Equivalent to growing by the length of s, then copying s into the new segment.
func (b *OutMessage) AppendString(s string) {
	sh := reflect.SliceHeader{
		Data: uintptr(b.Grow(uintptr(len(s)))),
		Len:  len(s),
		Cap:  len(s),
	}

	copy(*(*[]byte)(unsafe.Pointer(&sh)), s)
}

// Return a reference to the current contents of the buffer.
func (b *OutMessage) Bytes() []byte {
	return b.slice
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poller

import (
	"context"
	"strconv"
	"testing"
	"time"
)

type numError struct {
	num int
}

func (e numError) Error() string { return strconv.Itoa(e.num) }

func Test(t *testing.T) {
	var goods, bads []int

	cur := -1
	getter := func(context.Context) (interface{}, error) {
		// Even: success; odd: failure.
		cur++
		if cur%2 == 0 {
			return cur, nil
		}
		return nil, numError{cur}
	}

	onError := func(err error) {
		bads = append(bads, err.(numError).num)
	}

	p := New(cur, getter, onError)
	if got, want := p.Current(), cur; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	ctx, cancel := context.WithCancel(context.Background())
	p.Start(ctx, 50*time.Millisecond)
	time.Sleep(100 * time.Millisecond) // wait for first poll
	for i := 0; i < 10; i++ {
		goods = append(goods, p.Current().(int))
		time.Sleep(60 * time.Millisecond)
	}
	cancel()
	// Expect goods to be all even and non-decreasing.
	for i, g := range goods {
		if g%2 != 0 || (i > 0 && goods[i-1] > g) {
			t.Errorf("incorrect 'good' value %d", g)
		}
	}
	// Expect bads to be consecutive odd numbers.
	for i, b := range bads {
		if b%2 == 0 || (i > 0 && bads[i-1]+2 != b) {
			t.Errorf("incorrect 'bad' value %d", b)
		}
	}
}
<commit_msg>internal/poller: fix data race in poller_test<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poller

import (
	"context"
	"strconv"
	"sync"
	"testing"
	"time"
)

type numError struct {
	num int
}

func (e numError) Error() string { return strconv.Itoa(e.num) }

func Test(t *testing.T) {
	var (
		mu          sync.Mutex
		goods, bads []int
	)

	cur := -1
	getter := func(context.Context) (interface{}, error) {
		// Even: success; odd: failure.
		cur++
		if cur%2 == 0 {
			return cur, nil
		}
		return nil, numError{cur}
	}

	onError := func(err error) {
		mu.Lock()
		bads = append(bads, err.(numError).num)
		mu.Unlock()
	}

	p := New(cur, getter, onError)
	if got, want := p.Current(), cur; got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
	ctx, cancel := context.WithCancel(context.Background())
	p.Start(ctx, 50*time.Millisecond)
	time.Sleep(100 * time.Millisecond) // wait for first poll
	for i := 0; i < 10; i++ {
		goods = append(goods, p.Current().(int))
		time.Sleep(60 * time.Millisecond)
	}
	cancel()
	// Expect goods to be all even and non-decreasing.
	for i, g := range goods {
		if g%2 != 0 || (i > 0 && goods[i-1] > g) {
			t.Errorf("incorrect 'good' value %d", g)
		}
	}
	// Expect bads to be consecutive odd numbers.
	mu.Lock()
	bs := bads
	mu.Unlock()
	for i, b := range bs {
		if b%2 == 0 || (i > 0 && bads[i-1]+2 != b) {
			t.Errorf("incorrect 'bad' value %d", b)
		}
	}
}
<|endoftext|>


<commit_before>// A framework that allows you to create your own web robots!
package gobot

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"runtime"
)

var (
	tagRe  = regexp.MustCompilePOSIX("<a[^>]*>")
	attrRe = regexp.MustCompilePOSIX("href=\"[^\"]*\"")
	urlRe  = regexp.MustCompilePOSIX("http://[^\"]*")
	imgRe = regexp.MustCompilePOSIX(".(gif|jpg|jpeg|png)$")
)

type VisitAction func(*http.Response)

type FoundAction func(string)

type ImageAction func(string)

type ErrorAction func(*http.Request)

type VisitDecision func(string) bool

type GoBot struct {
	http.Client
	OnVisit     VisitAction
    OnFind      FoundAction
    OnImage     ImageAction
	OnError     ErrorAction
	ShouldVisit VisitDecision
	visited     map[string]bool
}

func NewGoBot() *GoBot {
	bot := new(GoBot)
	bot.OnVisit = defaultVisitAction
    bot.OnFind = defaultFoundAction
	bot.OnError = defaultErrorAction
	bot.ShouldVisit = defaultVisitDecision
	bot.OnImage = defaultImageAction
	bot.Jar = NewBotCookieJar()
	bot.visited = make(map[string]bool)
	return bot
}

func (bot *GoBot) StartCrawl(seed string) {
	resp, err := bot.Get(seed)
	ncpu := runtime.NumCPU()
	runtime.GOMAXPROCS(ncpu)
	if err != nil {
		log.Printf("StartCrawl: ERROR\n")
	}
	urls := ExtractLinks(resp)
	for i, u := range urls {
		if i == (ncpu - 1) {
			break
		}
		go bot.Crawl(u)
	}
	bot.Crawl(seed)
}

func (bot *GoBot) Crawl(seed string) {
	queue := make([]string, 0)
	currUrl := seed

	for {
		resp, err := bot.Get(currUrl)
		for err != nil {
			if resp != nil {
				go bot.OnError(resp.Request)
			}
			if len(queue) > 0 {
				currUrl = queue[0]
				queue = queue[1:]
				resp, err = bot.Get(currUrl)
			} else {
				os.Exit(1) // TODO: Should find a better way to exit.
			}
		}

		go bot.OnVisit(resp)

		urls := ExtractLinks(resp)
		for _, url := range urls {
			_, present := bot.visited[url]
			if !present {
                go bot.OnFind(url)
                if IsImage(url) {
                	go bot.OnImage(url)
                } else {
					queue = append(queue, url)
				}
				bot.visited[url] = true
			}
		}

		if len(queue) > 0 {
			currUrl = queue[0]
			queue = queue[1:]
		} else {
			break
		}

	}
}

// Extracts all the links from the body of an http response.
func ExtractLinks(resp *http.Response) []string {
	urls := make([]string, 0)
	body := ResponseBodyToString(resp)
	tags := tagRe.FindAllString(body, -1)

	for _, tag := range tags {
		url := urlRe.FindString(attrRe.FindString(tag))
		if url != "" {
			urls = append(urls, url)
		}
	}
	return urls
}

// Returns a string of the body of an http response.
func ResponseBodyToString(resp *http.Response) string {
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ""
	}
	return string(body)
}

func IsImage(url string) bool {
	return imgRe.MatchString(url)
}

func defaultVisitAction(resp *http.Response) {
	request := resp.Request
	log.Printf("%s\n", request.URL.String())
}

func defaultFoundAction(u string) {
    //log.Printf("%s\n", u)
}

func defaultImageAction(u string) {
	//log.Printf("%s\n", u)
}

func defaultErrorAction(resp *http.Request) {
}

func defaultVisitDecision(url string) bool {
	return true
}

type BotCookieJar struct {
	cookies map[string][]*http.Cookie
}

func NewBotCookieJar() *BotCookieJar {
	jar := new(BotCookieJar)
	jar.cookies = make(map[string][]*http.Cookie)
	return jar
}

func (jar *BotCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
	jar.cookies[u.Host] = cookies
}

func (jar *BotCookieJar) Cookies(u *url.URL) []*http.Cookie {
	return jar.cookies[u.Host]
}
<commit_msg>Run go fmt<commit_after>// A framework that allows you to create your own web robots!
package gobot

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"runtime"
)

var (
	tagRe  = regexp.MustCompilePOSIX("<a[^>]*>")
	attrRe = regexp.MustCompilePOSIX("href=\"[^\"]*\"")
	urlRe  = regexp.MustCompilePOSIX("http://[^\"]*")
	imgRe  = regexp.MustCompilePOSIX(".(gif|jpg|jpeg|png)$")
)

type VisitAction func(*http.Response)

type FoundAction func(string)

type ImageAction func(string)

type ErrorAction func(*http.Request)

type VisitDecision func(string) bool

type GoBot struct {
	http.Client
	OnVisit     VisitAction
	OnFind      FoundAction
	OnImage     ImageAction
	OnError     ErrorAction
	ShouldVisit VisitDecision
	visited     map[string]bool
}

func NewGoBot() *GoBot {
	bot := new(GoBot)
	bot.OnVisit = defaultVisitAction
	bot.OnFind = defaultFoundAction
	bot.OnError = defaultErrorAction
	bot.ShouldVisit = defaultVisitDecision
	bot.OnImage = defaultImageAction
	bot.Jar = NewBotCookieJar()
	bot.visited = make(map[string]bool)
	return bot
}

func (bot *GoBot) StartCrawl(seed string) {
	resp, err := bot.Get(seed)
	ncpu := runtime.NumCPU()
	runtime.GOMAXPROCS(ncpu)
	if err != nil {
		log.Printf("StartCrawl: ERROR\n")
	}
	urls := ExtractLinks(resp)
	for i, u := range urls {
		if i == (ncpu - 1) {
			break
		}
		go bot.Crawl(u)
	}
	bot.Crawl(seed)
}

func (bot *GoBot) Crawl(seed string) {
	queue := make([]string, 0)
	currUrl := seed

	for {
		resp, err := bot.Get(currUrl)
		for err != nil {
			if resp != nil {
				go bot.OnError(resp.Request)
			}
			if len(queue) > 0 {
				currUrl = queue[0]
				queue = queue[1:]
				resp, err = bot.Get(currUrl)
			} else {
				os.Exit(1) // TODO: Should find a better way to exit.
			}
		}

		go bot.OnVisit(resp)

		urls := ExtractLinks(resp)
		for _, url := range urls {
			_, present := bot.visited[url]
			if !present {
				go bot.OnFind(url)
				if IsImage(url) {
					go bot.OnImage(url)
				} else {
					queue = append(queue, url)
				}
				bot.visited[url] = true
			}
		}

		if len(queue) > 0 {
			currUrl = queue[0]
			queue = queue[1:]
		} else {
			break
		}

	}
}

// Extracts all the links from the body of an http response.
func ExtractLinks(resp *http.Response) []string {
	urls := make([]string, 0)
	body := ResponseBodyToString(resp)
	tags := tagRe.FindAllString(body, -1)

	for _, tag := range tags {
		url := urlRe.FindString(attrRe.FindString(tag))
		if url != "" {
			urls = append(urls, url)
		}
	}
	return urls
}

// Returns a string of the body of an http response.
func ResponseBodyToString(resp *http.Response) string {
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ""
	}
	return string(body)
}

func IsImage(url string) bool {
	return imgRe.MatchString(url)
}

func defaultVisitAction(resp *http.Response) {
	request := resp.Request
	log.Printf("%s\n", request.URL.String())
}

func defaultFoundAction(u string) {
	//log.Printf("%s\n", u)
}

func defaultImageAction(u string) {
	//log.Printf("%s\n", u)
}

func defaultErrorAction(resp *http.Request) {
}

func defaultVisitDecision(url string) bool {
	return true
}

type BotCookieJar struct {
	cookies map[string][]*http.Cookie
}

func NewBotCookieJar() *BotCookieJar {
	jar := new(BotCookieJar)
	jar.cookies = make(map[string][]*http.Cookie)
	return jar
}

func (jar *BotCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
	jar.cookies[u.Host] = cookies
}

func (jar *BotCookieJar) Cookies(u *url.URL) []*http.Cookie {
	return jar.cookies[u.Host]
}
<|endoftext|>


<commit_before>package webobs

import (
	"code.google.com/p/go.net/websocket"
	"fmt"
	"io"
	"net/http"
	"sync"
)

type Message struct {
	tag  string
	data []byte
}

type Client struct {
	readCh  chan Message
	writeCh chan []byte
	ws      *websocket.Conn
	tag     string
	id      int
}

type Server struct {
	clients   map[string][]*Client
	clientRCh chan Message
	writeCh   chan Message
	listeners map[string][]Listener
	mutex     *sync.Mutex

}

type Listener func(tag string, data []byte)

func (s *Server) SetChannel(tag string, l Listener, path string) {
	s.setHandler(tag, path)
	if l != nil {
		s.mutex.Lock()
		if _, ok := s.listeners[tag]; !ok {
			s.listeners[tag] = make([]Listener, 0)
		}
		s.listeners[tag] = append(s.listeners[tag], l)
		s.mutex.Unlock()
	}
}

func (s *Server) addNewClient(tag string, ws *websocket.Conn) *Client {

	s.mutex.Lock()
	if _, ok := s.clients[tag]; !ok {
		s.clients[tag] = make([]*Client, 0)
	}
	s.mutex.Unlock()

	cid := len(s.clients[tag])
	wc := make(chan []byte)
	c := Client{readCh: s.clientRCh, writeCh: wc, ws: ws, tag: tag, id: cid}
	s.mutex.Lock()
	s.clients[tag] = append(s.clients[tag], &c)
	s.mutex.Unlock()
	return &c
}

func (s *Server) removeClient(tag string, id int) {
	s.mutex.Lock()
	if _, ok := s.clients[tag]; !ok {
		return
	}
	s.mutex.Unlock()
	var idx int = -1
	for pos, c := range s.clients[tag] {
		if c.id == id {
			idx = pos
			break
		}
	}
	if idx == -1 {
		return
	}
	s.mutex.Lock()
	cl := s.clients[tag]
	s.clients[tag] = append(cl[:idx], cl[idx+1:]...)
	s.mutex.Unlock()
}

func (s *Server) listen() {
	// listen to clients
	go func() {
		for {
			select {
			case cmsg := <-s.clientRCh:
				fmt.Println("got message from client", cmsg)
				// check listener
				if _, ok := s.listeners[cmsg.tag]; ok {
					for _, listener := range s.listeners[cmsg.tag] {
						listener(cmsg.tag, cmsg.data)
					}
				}
			}
		}
	}()

	// listen to app
	func() {
		for {
			select {
			case amsg := <-s.writeCh:
				fmt.Println("got message from app", amsg)
				if _, ok := s.clients[amsg.tag]; ok {
					for _, cch := range s.clients[amsg.tag] {
						cch.writeCh <- amsg.data
					}
				}
			}
		}
	}()
}

func registerWS(tag string, ws *websocket.Conn, s *Server) {
	fmt.Println("register", tag)

	c := s.addNewClient(tag, ws)
	defer func() {
		s.removeClient(tag, c.id)
	}()

	go c.listenWrite()
	c.listenRead()
}

func (c *Client) listenRead() {
	for {
		select {
		default:
			var data []byte
			err := websocket.Message.Receive(c.ws, &data)
			if err == io.EOF {
				// close client
				return
			} else if err != nil {

			} else {
				c.readCh <- Message{tag: c.tag, data: data}
			}
		}
	}
}

func (c *Client) listenWrite() {
	for {
		select {
		case amsg := <-c.writeCh:
			err := websocket.Message.Send(c.ws, string(amsg))
			if err != nil {
				fmt.Println("send to client pb", amsg, err)
			}
		}
	}
}

func (s *Server) setHandler(tag string, scriptpath string) {
	tagws := tag + "_ws"
	clientscriptpath := "/" + tag + "_res/"

	http.Handle(
		clientscriptpath,
		http.StripPrefix(
			clientscriptpath,
			http.FileServer(http.Dir(scriptpath))))

	http.Handle("/"+tagws, websocket.Handler(
		func(ws *websocket.Conn) {
			registerWS(tag, ws, s)
		}))

	http.HandleFunc("/"+tag, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "
%s
"+
			""+
			"",
			tag, tagws, clientscriptpath, tag+".js")
	})
}

func newServer() *Server {
	cs := make(map[string][]*Client)
	crc := make(chan Message)
	wc := make(chan Message)
	l := make(map[string][]Listener)
	mx := &sync.Mutex{}
	return &Server{clients: cs, clientRCh: crc, writeCh: wc, listeners: l, mutex: mx}
}

func StartServer(port string) *Server {
	s := newServer()

	go func() {
		err := http.ListenAndServe(port, nil)
		if err != nil {
			panic("ListenAndServe: " + err.Error())
		}
	}()

	go s.listen()

	return s
}
<commit_msg>fix exports<commit_after>package webobs

import (
	"code.google.com/p/go.net/websocket"
	"fmt"
	"io"
	"net/http"
	"sync"
)

type Message struct {
	Tag  string
	Data []byte
}

type clientobs struct {
	readCh  chan Message
	writeCh chan []byte
	ws      *websocket.Conn
	tag     string
	id      int
}

type Server struct {
	clients   map[string][]*clientobs
	clientRCh chan Message
	WriteCh   chan Message
	listeners map[string][]Listener
	mutex     *sync.Mutex

}

type Listener func(tag string, data []byte)

func (s *Server) SetChannel(tag string, l Listener, path string) {
	s.setHandler(tag, path)
	if l != nil {
		s.mutex.Lock()
		if _, ok := s.listeners[tag]; !ok {
			s.listeners[tag] = make([]Listener, 0)
		}
		s.listeners[tag] = append(s.listeners[tag], l)
		s.mutex.Unlock()
	}
}

func (s *Server) addNewclientobs(tag string, ws *websocket.Conn) *clientobs {

	s.mutex.Lock()
	if _, ok := s.clients[tag]; !ok {
		s.clients[tag] = make([]*clientobs, 0)
	}
	s.mutex.Unlock()

	cid := len(s.clients[tag])
	wc := make(chan []byte)
	c := clientobs{readCh: s.clientRCh, writeCh: wc, ws: ws, tag: tag, id: cid}
	s.mutex.Lock()
	s.clients[tag] = append(s.clients[tag], &c)
	s.mutex.Unlock()
	return &c
}

func (s *Server) removeclientobs(tag string, id int) {
	s.mutex.Lock()
	if _, ok := s.clients[tag]; !ok {
		return
	}
	s.mutex.Unlock()
	var idx int = -1
	for pos, c := range s.clients[tag] {
		if c.id == id {
			idx = pos
			break
		}
	}
	if idx == -1 {
		return
	}
	s.mutex.Lock()
	cl := s.clients[tag]
	s.clients[tag] = append(cl[:idx], cl[idx+1:]...)
	s.mutex.Unlock()
}

func (s *Server) listen() {
	// listen to clients
	go func() {
		for {
			select {
			case cmsg := <-s.clientRCh:
				fmt.Println("got message from client", cmsg)
				// check listener
				if _, ok := s.listeners[cmsg.Tag]; ok {
					for _, listener := range s.listeners[cmsg.Tag] {
						listener(cmsg.Tag, cmsg.Data)
					}
				}
			}
		}
	}()

	// listen to app
	func() {
		for {
			select {
			case amsg := <-s.WriteCh:
				fmt.Println("got message from app", amsg)
				if _, ok := s.clients[amsg.Tag]; ok {
					for _, cch := range s.clients[amsg.Tag] {
						cch.writeCh <- amsg.Data
					}
				}
			}
		}
	}()
}

func registerWS(tag string, ws *websocket.Conn, s *Server) {
	fmt.Println("register", tag)

	c := s.addNewclientobs(tag, ws)
	defer func() {
		s.removeclientobs(tag, c.id)
	}()

	go c.listenWrite()
	c.listenRead()
}

func (c *clientobs) listenRead() {
	for {
		select {
		default:
			var data []byte
			err := websocket.Message.Receive(c.ws, &data)
			if err == io.EOF {
				// close client
				return
			} else if err != nil {

			} else {
				c.readCh <- Message{Tag: c.tag, Data: data}
			}
		}
	}
}

func (c *clientobs) listenWrite() {
	for {
		select {
		case amsg := <-c.writeCh:
			err := websocket.Message.Send(c.ws, string(amsg))
			if err != nil {
				fmt.Println("send to client pb", amsg, err)
			}
		}
	}
}

func (s *Server) setHandler(tag string, scriptpath string) {
	tagws := tag + "_ws"
	clientscriptpath := "/" + tag + "_res/"

	http.Handle(
		clientscriptpath,
		http.StripPrefix(
			clientscriptpath,
			http.FileServer(http.Dir(scriptpath))))

	http.Handle("/"+tagws, websocket.Handler(
		func(ws *websocket.Conn) {
			registerWS(tag, ws, s)
		}))

	http.HandleFunc("/"+tag, func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "
%s
"+
			""+
			"",
			tag, tagws, clientscriptpath, tag+".js")
	})
}

func newServer() *Server {
	cs := make(map[string][]*clientobs)
	crc := make(chan Message)
	wc := make(chan Message)
	l := make(map[string][]Listener)
	mx := &sync.Mutex{}
	return &Server{clients: cs, clientRCh: crc, WriteCh: wc, listeners: l, mutex: mx}
}

func StartServer(port string) *Server {
	s := newServer()

	go func() {
		err := http.ListenAndServe(port, nil)
		if err != nil {
			panic("ListenAndServe: " + err.Error())
		}
	}()

	go s.listen()

	return s
}
<|endoftext|>