<commit_before>package public

import (
	"bitbucket.org/ikeikeikeike/antenna/models"

	"github.com/astaxie/beego/utils/pagination"
	// "github.com/k0kubun/pp"
)

type DivasController struct {
	BaseController
}

func (c *DivasController) NestFinish() {
	c.PushInAccessLog()
}

func (c *DivasController) Index() {
	c.TplNames = "public/divas/index.tpl"

	pers := c.DefaultPers
	qs := models.Divas().RelatedSel()
	qs = c.SetBracupLines(c.SetPrefixLines(qs.Filter("videos_count__gt", 0), ""))

	cnt, _ := models.CountObjects(qs)
	pager := pagination.SetPaginator(c.Ctx, pers, cnt)

	qs = qs.OrderBy("-videos_count").Limit(pers, pager.Offset())

	var divas []*models.Diva
	models.ListObjects(qs, &divas)

	c.Data["Divas"] = divas
}

func (c *DivasController) Show() {
	c.TplNames = "public/divas/show.tpl"

	name := c.Ctx.Input.Param(":name")
	if name == "" {
		c.Ctx.Abort(404, "404 NotFound")
		return
	}

	s := &models.Diva{Name: name}
	s.Read("Name")

	if s.Id <= 0 {
		c.Ctx.Abort(404, "404 NotFound")
		return
	}

	s.LoadRelated()
	c.Data["Diva"] = s

	pers := c.DefaultPers
	qs := models.Videos()
	qs = qs.Filter("divas__diva__name", name)
	qs = qs.Filter("entry__isnull", false)

	cnt, _ := models.CountObjects(qs)
	pager := pagination.SetPaginator(c.Ctx, pers, cnt)

	qs = qs.Limit(pers, pager.Offset()).RelatedSel()

	var videos []*models.Video
	models.ListObjects(qs, &videos)

	c.Data["Videos"] = videos
}
<commit_msg>caching to the diva,anime,character page<commit_after>package public

import (
	"time"

	"bitbucket.org/ikeikeikeike/antenna/models"

	"github.com/astaxie/beego/utils/pagination"
	// "github.com/k0kubun/pp"
)

type DivasController struct {
	BaseController
}

func (c *DivasController) NestFinish() {
	c.PushInAccessLog()
}

func (c *DivasController) Index() {
	c.TplNames = "public/divas/index.tpl"

	pers := c.DefaultPers
	qs := models.Divas().RelatedSel()
	qs = c.SetBracupLines(c.SetPrefixLines(qs.Filter("videos_count__gt", 0), ""))

	cnt, _ := models.CountObjects(qs)
	pager := pagination.SetPaginator(c.Ctx, pers, cnt)

	qs = qs.OrderBy("-videos_count").Limit(pers, pager.Offset())

	var divas []*models.Diva
	models.ListObjects(qs, &divas)

	c.Data["Divas"] = divas
}

func (c *DivasController) Show() {
	c.TplNames = "public/divas/show.tpl"

	name := c.Ctx.Input.Param(":name")
	if name == "" {
		c.Ctx.Abort(404, "404 NotFound")
		return
	}

	s := &models.Diva{Name: name}
	s.Read("Name")

	if s.Id <= 0 {
		c.Ctx.Abort(404, "404 NotFound")
		return
	}

	// Update raw html
	if c.IsAjax() && c.Ctx.Input.IsPost() {
		s.Html = c.GetString("data")
		s.Update("Html", "Updated")
		c.ServeJson()
		return
	}
	// Raw html update expire: 5 day
	if (s.Updated.Unix() + (60 * 60 * 24 * 5)) < time.Now().Unix() {
		s.Html = ""
	}
	c.Data["xsrftoken"] = c.XsrfToken()

	s.LoadRelated()
	c.Data["Diva"] = s

	pers := c.DefaultPers
	qs := models.Videos()
	qs = qs.Filter("divas__diva__name", name)
	qs = qs.Filter("entry__isnull", false)

	cnt, _ := models.CountObjects(qs)
	pager := pagination.SetPaginator(c.Ctx, pers, cnt)

	qs = qs.Limit(pers, pager.Offset()).RelatedSel()

	var videos []*models.Video
	models.ListObjects(qs, &videos)

	c.Data["Videos"] = videos
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package probes

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lebauce/dockerclient"

	"github.com/redhat-cip/skydive/config"
	"github.com/redhat-cip/skydive/logging"
	"github.com/redhat-cip/skydive/topology/graph"
)

const (
	StoppedState  = iota
	RunningState  = iota
	StoppingState = iota
)

type DockerProbe struct {
	sync.RWMutex
	NetNSProbe
	url       string
	client    *dockerclient.DockerClient
	state     int64
	connected atomic.Value
	quit      chan bool
	wg        sync.WaitGroup
	idToPid   map[string]int
}

type DockerContainerAttributes struct {
	ContainerID string
}

func (probe *DockerProbe) containerNamespace(pid int) string {
	return fmt.Sprintf("/proc/%d/ns/net", pid)
}

func (probe *DockerProbe) registerContainer(id string) {
	probe.Lock()
	defer probe.Unlock()

	if _, ok := probe.idToPid[id]; ok {
		return
	}
	info, err := probe.client.InspectContainer(id)
	if err != nil {
		logging.GetLogger().Errorf("Failed to inspect Docker container %s: %s", id, err.Error())
		return
	}
	namespace := probe.containerNamespace(info.State.Pid)
	logging.GetLogger().Debugf("Register docker container %s and PID %d", info.Id, info.State.Pid)
	metadata := &graph.Metadata{
		"Name":                 info.Name[1:],
		"Manager":              "docker",
		"Docker.ContainerID":   info.Id,
		"Docker.ContainerName": info.Name,
		"Docker.ContainerPID":  info.State.Pid,
	}
	probe.Register(namespace, metadata)
	probe.idToPid[info.Id] = info.State.Pid
}

func (probe *DockerProbe) unregisterContainer(id string) {
	probe.Lock()
	defer probe.Unlock()

	pid, ok := probe.idToPid[id]
	if !ok {
		return
	}
	delete(probe.idToPid, id)
	namespace := probe.containerNamespace(pid)
	logging.GetLogger().Debugf("Stop listening for namespace %s with PID %d", namespace, pid)
	probe.Unregister(namespace)
}

func (probe *DockerProbe) handleDockerEvent(event *dockerclient.Event) {
	if event.Status == "start" {
		probe.registerContainer(event.ID)
	} else if event.Status == "die" {
		probe.unregisterContainer(event.ID)
	}
}

func (probe *DockerProbe) connect() error {
	var err error

	logging.GetLogger().Debugf("Connecting to Docker daemon: %s", probe.url)
	probe.client, err = dockerclient.NewDockerClient(probe.url, nil)
	if err != nil {
		logging.GetLogger().Errorf("Failed to connect to Docker daemon: %s", err.Error())
		return err
	}

	eventsOptions := &dockerclient.MonitorEventsOptions{
		Filters: &dockerclient.MonitorEventsFilters{
			Events: []string{"start", "die"},
		},
	}

	eventErrChan, err := probe.client.MonitorEvents(eventsOptions, nil)
	if err != nil {
		logging.GetLogger().Errorf("Unable to monitor Docker events: %s", err.Error())
		return err
	}

	probe.wg.Add(2)
	probe.quit = make(chan bool)

	probe.connected.Store(true)
	defer probe.connected.Store(false)

	go func() {
		defer probe.wg.Done()

		containers, err := probe.client.ListContainers(false, false, "")
		if err != nil {
			logging.GetLogger().Errorf("Failed to list containers: %s", err.Error())
			return
		}

		for _, c := range containers {
			if atomic.LoadInt64(&probe.state) != RunningState {
				break
			}
			probe.registerContainer(c.Id)
		}
	}()

	defer probe.wg.Done()

	for {
		select {
		case <-probe.quit:
			return nil
		case e := <-eventErrChan:
			if e.Error != nil {
				logging.GetLogger().Errorf("Got error while waiting for Docker event: %s", e.Error.Error())
				return e.Error
			}
			probe.handleDockerEvent(&e.Event)
		}
	}
}

func (probe *DockerProbe) Start() {
	if !atomic.CompareAndSwapInt64(&probe.state, StoppedState, RunningState) {
		return
	}

	go func() {
		for {
			state := atomic.LoadInt64(&probe.state)
			if state == StoppingState || state == StoppedState {
				break
			}

			if probe.connect() != nil {
				time.Sleep(1 * time.Second)
			}
		}
	}()
}

func (probe *DockerProbe) Stop() {
	if !atomic.CompareAndSwapInt64(&probe.state, RunningState, StoppingState) {
		return
	}

	if probe.connected.Load() == true {
		close(probe.quit)
		probe.wg.Wait()
	}

	atomic.StoreInt64(&probe.state, StoppedState)
}

func NewDockerProbe(g *graph.Graph, n *graph.Node, dockerURL string) (probe *DockerProbe) {
	probe = &DockerProbe{
		NetNSProbe: *NewNetNSProbe(g, n),
		url:        dockerURL,
		idToPid:    make(map[string]int),
		state:      StoppedState,
	}
	return
}

func NewDockerProbeFromConfig(g *graph.Graph, n *graph.Node) *DockerProbe {
	dockerURL := config.GetConfig().GetString("docker.url")
	return NewDockerProbe(g, n, dockerURL)
}
<commit_msg>Ignore Docker containers in net=host mode<commit_after>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package probes

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lebauce/dockerclient"
	"github.com/vishvananda/netns"

	"github.com/redhat-cip/skydive/config"
	"github.com/redhat-cip/skydive/logging"
	"github.com/redhat-cip/skydive/topology/graph"
)

const (
	StoppedState  = iota
	RunningState  = iota
	StoppingState = iota
)

type DockerProbe struct {
	sync.RWMutex
	NetNSProbe
	url       string
	client    *dockerclient.DockerClient
	state     int64
	connected atomic.Value
	quit      chan bool
	wg        sync.WaitGroup
	idToPid   map[string]int
	hostNs    netns.NsHandle
}

type DockerContainerAttributes struct {
	ContainerID string
}

func (probe *DockerProbe) containerNamespace(pid int) string {
	return fmt.Sprintf("/proc/%d/ns/net", pid)
}

func (probe *DockerProbe) registerContainer(id string) {
	probe.Lock()
	defer probe.Unlock()

	if _, ok := probe.idToPid[id]; ok {
		return
	}
	info, err := probe.client.InspectContainer(id)
	if err != nil {
		logging.GetLogger().Errorf("Failed to inspect Docker container %s: %s", id, err.Error())
		return
	}

	nsHandle, err := netns.GetFromPid(info.State.Pid)
	if err != nil {
		return
	}

	if probe.hostNs.Equal(nsHandle) {
		// The container is in net=host mode
		return
	}

	namespace := probe.containerNamespace(info.State.Pid)
	logging.GetLogger().Debugf("Register docker container %s and PID %d", info.Id, info.State.Pid)
	metadata := &graph.Metadata{
		"Name":                 info.Name[1:],
		"Manager":              "docker",
		"Docker.ContainerID":   info.Id,
		"Docker.ContainerName": info.Name,
		"Docker.ContainerPID":  info.State.Pid,
	}
	probe.Register(namespace, metadata)
	probe.idToPid[info.Id] = info.State.Pid
}

func (probe *DockerProbe) unregisterContainer(id string) {
	probe.Lock()
	defer probe.Unlock()

	pid, ok := probe.idToPid[id]
	if !ok {
		return
	}
	delete(probe.idToPid, id)
	namespace := probe.containerNamespace(pid)
	logging.GetLogger().Debugf("Stop listening for namespace %s with PID %d", namespace, pid)
	probe.Unregister(namespace)
}

func (probe *DockerProbe) handleDockerEvent(event *dockerclient.Event) {
	if event.Status == "start" {
		probe.registerContainer(event.ID)
	} else if event.Status == "die" {
		probe.unregisterContainer(event.ID)
	}
}

func (probe *DockerProbe) connect() error {
	var err error

	if probe.hostNs, err = netns.Get(); err != nil {
		return err
	}

	logging.GetLogger().Debugf("Connecting to Docker daemon: %s", probe.url)
	probe.client, err = dockerclient.NewDockerClient(probe.url, nil)
	if err != nil {
		logging.GetLogger().Errorf("Failed to connect to Docker daemon: %s", err.Error())
		return err
	}

	eventsOptions := &dockerclient.MonitorEventsOptions{
		Filters: &dockerclient.MonitorEventsFilters{
			Events: []string{"start", "die"},
		},
	}

	eventErrChan, err := probe.client.MonitorEvents(eventsOptions, nil)
	if err != nil {
		logging.GetLogger().Errorf("Unable to monitor Docker events: %s", err.Error())
		return err
	}

	probe.wg.Add(2)
	probe.quit = make(chan bool)

	probe.connected.Store(true)
	defer probe.connected.Store(false)

	go func() {
		defer probe.wg.Done()

		containers, err := probe.client.ListContainers(false, false, "")
		if err != nil {
			logging.GetLogger().Errorf("Failed to list containers: %s", err.Error())
			return
		}

		for _, c := range containers {
			if atomic.LoadInt64(&probe.state) != RunningState {
				break
			}
			probe.registerContainer(c.Id)
		}
	}()

	defer probe.wg.Done()

	for {
		select {
		case <-probe.quit:
			return nil
		case e := <-eventErrChan:
			if e.Error != nil {
				logging.GetLogger().Errorf("Got error while waiting for Docker event: %s", e.Error.Error())
				return e.Error
			}
			probe.handleDockerEvent(&e.Event)
		}
	}
}

func (probe *DockerProbe) Start() {
	if !atomic.CompareAndSwapInt64(&probe.state, StoppedState, RunningState) {
		return
	}

	go func() {
		for {
			state := atomic.LoadInt64(&probe.state)
			if state == StoppingState || state == StoppedState {
				break
			}

			if probe.connect() != nil {
				time.Sleep(1 * time.Second)
			}
		}
	}()
}

func (probe *DockerProbe) Stop() {
	if !atomic.CompareAndSwapInt64(&probe.state, RunningState, StoppingState) {
		return
	}

	if probe.connected.Load() == true {
		close(probe.quit)
		probe.wg.Wait()
	}

	atomic.StoreInt64(&probe.state, StoppedState)
}

func NewDockerProbe(g *graph.Graph, n *graph.Node, dockerURL string) (probe *DockerProbe) {
	probe = &DockerProbe{
		NetNSProbe: *NewNetNSProbe(g, n),
		url:        dockerURL,
		idToPid:    make(map[string]int),
		state:      StoppedState,
	}
	return
}

func NewDockerProbeFromConfig(g *graph.Graph, n *graph.Node) *DockerProbe {
	dockerURL := config.GetConfig().GetString("docker.url")
	return NewDockerProbe(g, n, dockerURL)
}
<|endoftext|>


<commit_before>// +build !safe
// +build !appengine

package xxhash

import (
	"reflect"
	"unsafe"
)

// Backend returns the current version of xxhash being used.
const Backend = "GoUnsafe"

// ChecksumString32S returns the checksum of the input data, without creating a copy, with the specific seed.
func ChecksumString32S(s string, seed uint32) uint32 {
	if len(s) == 0 {
		return Checksum32S(nil, seed)
	}
	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return Checksum32S((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)], seed)
}

func (xx *XXHash32) WriteString(s string) (int, error) {
	if len(s) == 0 {
		return 0, nil
	}

	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return xx.Write((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)])
}

// ChecksumString64S returns the checksum of the input data, without creating a copy, with the specific seed.
func ChecksumString64S(s string, seed uint64) uint64 {
	if len(s) == 0 {
		return Checksum64S(nil, seed)
	}

	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return Checksum64S((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)], seed)
}

func (xx *XXHash64) WriteString(s string) (int, error) {
	if len(s) == 0 {
		return 0, nil
	}
	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return xx.Write((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s)])
}

func checksum64(in []byte, seed uint64) uint64 {
	var (
		wordsLen = len(in) >> 3
		words    = ((*[maxInt32]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen]

		h uint64 = prime64x5

		v1, v2, v3, v4 = resetVs64(seed)

		i int
	)

	for ; i < len(words)-3; i += 4 {
		words := (*[4]uint64)(unsafe.Pointer(&words[i]))

		v1 = round64(v1, words[0])
		v2 = round64(v2, words[1])
		v3 = round64(v3, words[2])
		v4 = round64(v4, words[3])
	}

	h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4)

	h = mergeRound64(h, v1)
	h = mergeRound64(h, v2)
	h = mergeRound64(h, v3)
	h = mergeRound64(h, v4)

	h += uint64(len(in))

	for _, k := range words[i:] {
		h ^= round64(0, k)
		h = rotl64_27(h)*prime64x1 + prime64x4
	}

	if in = in[wordsLen<<3 : len(in):len(in)]; len(in) > 3 {
		words := (*[1]uint32)(unsafe.Pointer(&in[0]))
		h ^= uint64(words[0]) * prime64x1
		h = rotl64_23(h)*prime64x2 + prime64x3

		in = in[4:len(in):len(in)]
	}

	for _, b := range in {
		h ^= uint64(b) * prime64x5
		h = rotl64_11(h) * prime64x1
	}

	return mix64(h)
}

func checksum64Short(in []byte, seed uint64) uint64 {
	var (
		h = seed + prime64x5 + uint64(len(in))
		i int
	)

	if len(in) > 7 {
		var (
			wordsLen = len(in) >> 3
			words    = ((*[maxInt32]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen]
		)

		for i := range words {
			h ^= round64(0, words[i])
			h = rotl64_27(h)*prime64x1 + prime64x4
		}

		i = wordsLen << 3
	}

	if in = in[i:len(in):len(in)]; len(in) > 3 {
		words := (*[1]uint32)(unsafe.Pointer(&in[0]))
		h ^= uint64(words[0]) * prime64x1
		h = rotl64_23(h)*prime64x2 + prime64x3

		in = in[4:len(in):len(in)]
	}

	for _, b := range in {
		h ^= uint64(b) * prime64x5
		h = rotl64_11(h) * prime64x1
	}

	return mix64(h)
}

func (xx *XXHash64) Write(in []byte) (n int, err error) {
	mem, idx := xx.mem[:], int(xx.memIdx)

	xx.ln, n = xx.ln+uint64(len(in)), len(in)

	if idx+len(in) < 32 {
		xx.memIdx += int8(copy(mem[idx:len(mem):len(mem)], in))
		return
	}

	var (
		v1, v2, v3, v4 = xx.v1, xx.v2, xx.v3, xx.v4

		i int
	)

	if d := 32 - int(idx); d > 0 && int(idx)+len(in) > 31 {
		copy(mem[idx:len(mem):len(mem)], in[:len(in):len(in)])

		words := (*[4]uint64)(unsafe.Pointer(&mem[0]))

		v1 = round64(v1, words[0])
		v2 = round64(v2, words[1])
		v3 = round64(v3, words[2])
		v4 = round64(v4, words[3])

		if in, xx.memIdx = in[d:len(in):len(in)], 0; len(in) == 0 {
			goto RET
		}
	}

	for ; i < len(in)-31; i += 32 {
		words := (*[4]uint64)(unsafe.Pointer(&in[i]))

		v1 = round64(v1, words[0])
		v2 = round64(v2, words[1])
		v3 = round64(v3, words[2])
		v4 = round64(v4, words[3])
	}

	if len(in)-i != 0 {
		xx.memIdx += int8(copy(mem[xx.memIdx:len(mem):len(mem)], in[i:len(in):len(in)]))
	}

RET:
	xx.v1, xx.v2, xx.v3, xx.v4 = v1, v2, v3, v4

	return
}

func (xx *XXHash64) Sum64() (h uint64) {
	if seed := xx.seed; xx.ln > 31 {
		v1, v2, v3, v4 := xx.v1, xx.v2, xx.v3, xx.v4
		h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4)

		h = mergeRound64(h, v1)
		h = mergeRound64(h, v2)
		h = mergeRound64(h, v3)
		h = mergeRound64(h, v4)
	} else if seed == 0 {
		h = prime64x5
	} else {
		h = seed + prime64x5
	}

	h += uint64(xx.ln)

	if xx.memIdx == 0 {
		return mix64(h)
	}

	var (
		in       = xx.mem[:xx.memIdx:xx.memIdx]
		wordsLen = len(in) >> 3
		words    = ((*[maxInt32]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen]
	)

	for _, k := range words {
		h ^= round64(0, k)
		h = rotl64_27(h)*prime64x1 + prime64x4
	}

	if in = in[wordsLen<<3 : len(in):len(in)]; len(in) > 3 {
		words := (*[1]uint32)(unsafe.Pointer(&in[0]))

		h ^= uint64(words[0]) * prime64x1
		h = rotl64_23(h)*prime64x2 + prime64x3

		in = in[4:len(in):len(in)]
	}

	for _, b := range in {
		h ^= uint64(b) * prime64x5
		h = rotl64_11(h) * prime64x1
	}

	return mix64(h)
}
<commit_msg>workaround  [2147483647]uint64 larger than address space on older 32bit archs.<commit_after>// +build !safe
// +build !appengine

package xxhash

import (
	"reflect"
	"unsafe"
)

// Backend returns the current version of xxhash being used.
const Backend = "GoUnsafe"

// ChecksumString32S returns the checksum of the input data, without creating a copy, with the specific seed.
func ChecksumString32S(s string, seed uint32) uint32 {
	if len(s) == 0 {
		return Checksum32S(nil, seed)
	}
	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return Checksum32S((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)], seed)
}

func (xx *XXHash32) WriteString(s string) (int, error) {
	if len(s) == 0 {
		return 0, nil
	}

	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return xx.Write((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)])
}

// ChecksumString64S returns the checksum of the input data, without creating a copy, with the specific seed.
func ChecksumString64S(s string, seed uint64) uint64 {
	if len(s) == 0 {
		return Checksum64S(nil, seed)
	}

	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return Checksum64S((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)], seed)
}

func (xx *XXHash64) WriteString(s string) (int, error) {
	if len(s) == 0 {
		return 0, nil
	}
	ss := (*reflect.StringHeader)(unsafe.Pointer(&s))
	return xx.Write((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s)])
}

func checksum64(in []byte, seed uint64) uint64 {
	var (
		wordsLen = len(in) >> 3
		words    = ((*[maxInt32 / 8]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen]

		h uint64 = prime64x5

		v1, v2, v3, v4 = resetVs64(seed)

		i int
	)

	for ; i < len(words)-3; i += 4 {
		words := (*[4]uint64)(unsafe.Pointer(&words[i]))

		v1 = round64(v1, words[0])
		v2 = round64(v2, words[1])
		v3 = round64(v3, words[2])
		v4 = round64(v4, words[3])
	}

	h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4)

	h = mergeRound64(h, v1)
	h = mergeRound64(h, v2)
	h = mergeRound64(h, v3)
	h = mergeRound64(h, v4)

	h += uint64(len(in))

	for _, k := range words[i:] {
		h ^= round64(0, k)
		h = rotl64_27(h)*prime64x1 + prime64x4
	}

	if in = in[wordsLen<<3 : len(in) : len(in)]; len(in) > 3 {
		words := (*[1]uint32)(unsafe.Pointer(&in[0]))
		h ^= uint64(words[0]) * prime64x1
		h = rotl64_23(h)*prime64x2 + prime64x3

		in = in[4:len(in):len(in)]
	}

	for _, b := range in {
		h ^= uint64(b) * prime64x5
		h = rotl64_11(h) * prime64x1
	}

	return mix64(h)
}

func checksum64Short(in []byte, seed uint64) uint64 {
	var (
		h = seed + prime64x5 + uint64(len(in))
		i int
	)

	if len(in) > 7 {
		var (
			wordsLen = len(in) >> 3
			words    = ((*[maxInt32 / 8]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen]
		)

		for i := range words {
			h ^= round64(0, words[i])
			h = rotl64_27(h)*prime64x1 + prime64x4
		}

		i = wordsLen << 3
	}

	if in = in[i:len(in):len(in)]; len(in) > 3 {
		words := (*[1]uint32)(unsafe.Pointer(&in[0]))
		h ^= uint64(words[0]) * prime64x1
		h = rotl64_23(h)*prime64x2 + prime64x3

		in = in[4:len(in):len(in)]
	}

	for _, b := range in {
		h ^= uint64(b) * prime64x5
		h = rotl64_11(h) * prime64x1
	}

	return mix64(h)
}

func (xx *XXHash64) Write(in []byte) (n int, err error) {
	mem, idx := xx.mem[:], int(xx.memIdx)

	xx.ln, n = xx.ln+uint64(len(in)), len(in)

	if idx+len(in) < 32 {
		xx.memIdx += int8(copy(mem[idx:len(mem):len(mem)], in))
		return
	}

	var (
		v1, v2, v3, v4 = xx.v1, xx.v2, xx.v3, xx.v4

		i int
	)

	if d := 32 - int(idx); d > 0 && int(idx)+len(in) > 31 {
		copy(mem[idx:len(mem):len(mem)], in[:len(in):len(in)])

		words := (*[4]uint64)(unsafe.Pointer(&mem[0]))

		v1 = round64(v1, words[0])
		v2 = round64(v2, words[1])
		v3 = round64(v3, words[2])
		v4 = round64(v4, words[3])

		if in, xx.memIdx = in[d:len(in):len(in)], 0; len(in) == 0 {
			goto RET
		}
	}

	for ; i < len(in)-31; i += 32 {
		words := (*[4]uint64)(unsafe.Pointer(&in[i]))

		v1 = round64(v1, words[0])
		v2 = round64(v2, words[1])
		v3 = round64(v3, words[2])
		v4 = round64(v4, words[3])
	}

	if len(in)-i != 0 {
		xx.memIdx += int8(copy(mem[xx.memIdx:len(mem):len(mem)], in[i:len(in):len(in)]))
	}

RET:
	xx.v1, xx.v2, xx.v3, xx.v4 = v1, v2, v3, v4

	return
}

func (xx *XXHash64) Sum64() (h uint64) {
	if seed := xx.seed; xx.ln > 31 {
		v1, v2, v3, v4 := xx.v1, xx.v2, xx.v3, xx.v4
		h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4)

		h = mergeRound64(h, v1)
		h = mergeRound64(h, v2)
		h = mergeRound64(h, v3)
		h = mergeRound64(h, v4)
	} else if seed == 0 {
		h = prime64x5
	} else {
		h = seed + prime64x5
	}

	h += uint64(xx.ln)

	if xx.memIdx == 0 {
		return mix64(h)
	}

	var (
		in       = xx.mem[:xx.memIdx:xx.memIdx]
		wordsLen = len(in) >> 3
		words    = ((*[maxInt32 / 8]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen]
	)

	for _, k := range words {
		h ^= round64(0, k)
		h = rotl64_27(h)*prime64x1 + prime64x4
	}

	if in = in[wordsLen<<3 : len(in) : len(in)]; len(in) > 3 {
		words := (*[1]uint32)(unsafe.Pointer(&in[0]))

		h ^= uint64(words[0]) * prime64x1
		h = rotl64_23(h)*prime64x2 + prime64x3

		in = in[4:len(in):len(in)]
	}

	for _, b := range in {
		h ^= uint64(b) * prime64x5
		h = rotl64_11(h) * prime64x1
	}

	return mix64(h)
}
<|endoftext|>


<commit_before>package main

import "flag"
import "fmt"

type Formula struct {
	value string
	left *Formula
	right *Formula
}

func parseFormula(f string) Formula {
	result := Formula{
		value: f,
		left: nil,
		right: nil,
	}
	return result
}

func printFormula(formula Formula) string {
    leftString := ""
    if formula.left != nil {
        leftString = printFormula(*formula.left)
    }
    rightString := ""
    if formula.right != nil {
        rightString = printFormula(*formula.right)
    }
    return "(" + leftString + formula.value + rightString + ")"
}

func main() {
	formulaString := flag.String("formula", "", "The formula in propositional logic")
	flag.Parse()
	formula := parseFormula(*formulaString)
	fmt.Println("Formula: " + formula.value)
}
<commit_msg>Adapted print function to hide parentheses for atomic formulas<commit_after>package main

import "flag"
import "fmt"

type Formula struct {
	value string
	left *Formula
	right *Formula
}

func parseFormula(f string) Formula {
	result := Formula{
		value: f,
		left: nil,
		right: nil,
	}
	return result
}

func printFormula(formula Formula) string {
    leftString := ""
    if formula.left != nil {
        leftString = printFormula(*formula.left)
    }
    rightString := ""
    if formula.right != nil {
        rightString = printFormula(*formula.right)
    }
    openString := "("
    closeString := ")"
    if formula.left == nil && formula.right == nil {
        openString = ""
        closeString = ""
    }
    return openString + leftString + formula.value + rightString + closeString
}

func main() {
	formulaString := flag.String("formula", "", "The formula in propositional logic")
	flag.Parse()
	formula := parseFormula(*formulaString)
	fmt.Println("Formula: " + formula.value)
}
<|endoftext|>


<commit_before>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| rpc/plugins/reverse/caller.go                            |
|                                                          |
| LastModified: May 19, 2021                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package reverse

import (
	"context"
	"errors"
	"reflect"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hprose/hprose-golang/v3/io"
	"github.com/hprose/hprose-golang/v3/rpc/core"
	"github.com/modern-go/reflect2"
	cmap "github.com/orcaman/concurrent-map"
)

type call [3]interface{}

func newCall(index int, name string, args []interface{}) (c call) {
	c[0] = index
	c[1] = name
	c[2] = args
	return
}

func (c call) Value() (index int, name string, args []interface{}) {
	return c[0].(int), c[1].(string), c[2].([]interface{})
}

type callCache struct {
	c []call
	sync.Mutex
}

func (cc *callCache) Append(c call) {
	cc.Lock()
	defer cc.Unlock()
	cc.c = append(cc.c, c)
}

func (cc *callCache) Delete(index int) {
	cc.Lock()
	defer cc.Unlock()
	for i := 0; i < len(cc.c); i++ {
		if cc.c[i][0].(int) == index {
			cc.c = append(cc.c[:i], cc.c[i+1:]...)
			return
		}
	}
}

func (cc *callCache) Take() (calls []call) {
	cc.Lock()
	defer cc.Unlock()
	calls = cc.c
	cc.c = nil
	return
}

type returnValue [3]interface{}

func newReturnValue(index int, result interface{}, err string) (r returnValue) {
	r[0] = index
	r[1] = result
	r[2] = err
	return
}

func (r returnValue) Index() int {
	return r[0].(int)
}

func (r returnValue) Value(returnType []reflect.Type) ([]interface{}, error) {
	err := r[2].(string)
	if err != "" {
		return nil, errors.New(err)
	}
	n := len(returnType)
	switch n {
	case 0:
		return nil, nil
	case 1:
		if result, err := io.Convert(r[1], returnType[0]); err != nil {
			return nil, err
		} else {
			return []interface{}{result}, nil
		}
	default:
		results := make([]interface{}, n)
		values := r[1].([]interface{})
		count := len(values)
		for i := 0; i < n && i < count; i++ {
			if result, err := io.Convert(values[i], returnType[i]); err != nil {
				return nil, err
			} else {
				results[i] = result
			}
		}
		for i := count; i < n; i++ {
			t := reflect2.Type2(returnType[i])
			results[i] = t.Indirect(t.New())
		}
		return results, nil
	}
}

type resultMap struct {
	results map[int]chan returnValue
	sync.Mutex
}

func newResultMap() *resultMap {
	return &resultMap{
		results: make(map[int]chan returnValue),
	}
}

func (m *resultMap) GetAndDelete(index int) chan returnValue {
	m.Lock()
	defer m.Unlock()
	if result, ok := m.results[index]; ok {
		delete(m.results, index)
		return result
	}
	return nil
}

func (m *resultMap) Delete(index int) {
	m.Lock()
	defer m.Unlock()
	delete(m.results, index)
}

func (m *resultMap) Set(index int, result chan returnValue) {
	m.Lock()
	defer m.Unlock()
	m.results[index] = result
}

var (
	emptyArgs = make([]interface{}, 0)
	emptyCall = make([]call, 0)
)

type Caller struct {
	*core.Service
	HeartBeat  time.Duration
	Timeout    time.Duration
	calls      cmap.ConcurrentMap
	results    cmap.ConcurrentMap
	responders cmap.ConcurrentMap
	onlines    cmap.ConcurrentMap
	counter    int32
}

func NewCaller(service *core.Service) *Caller {
	caller := &Caller{
		Service:    service,
		HeartBeat:  time.Minute * 2,
		Timeout:    time.Second * 30,
		calls:      cmap.New(),
		results:    cmap.New(),
		responders: cmap.New(),
		onlines:    cmap.New(),
	}
	service.Use(caller.handler).
		AddFunction(caller.close, "!!").
		AddFunction(caller.begin, "!").
		AddFunction(caller.end, "=")
	return caller
}

func (c *Caller) ID(ctx context.Context) (id string) {
	if id = core.GetServiceContext(ctx).RequestHeaders().GetString("id"); id == "" {
		panic("client unique id not found")
	}
	return
}

func (c *Caller) send(id string, responder chan []call) bool {
	if calls, ok := c.calls.Get(id); ok {
		calls := calls.(*callCache).Take()
		if len(calls) == 0 {
			return false
		}
		responder <- calls
		return true
	}
	return false
}

func (c *Caller) response(id string) {
	if responder, ok := c.responders.Pop(id); ok {
		responder := responder.(chan []call)
		if !c.send(id, responder) {
			if c.responders.SetIfAbsent(id, responder) {
				responder <- nil
			}
		}
	}
}

func (c *Caller) stop(ctx context.Context) string {
	id := c.ID(ctx)
	if responder, ok := c.responders.Pop(id); ok {
		responder.(chan []call) <- nil
	}
	return id
}

func (c *Caller) close(ctx context.Context) {
	id := c.stop(ctx)
	c.onlines.Remove(id)
}

func (c *Caller) begin(ctx context.Context) []call {
	id := c.stop(ctx)
	c.onlines.Set(id, true)
	responder := make(chan []call, 1)
	if !c.send(id, responder) {
		c.responders.Upsert(id, responder, func(exist bool, valueInMap interface{}, newValue interface{}) interface{} {
			if exist {
				valueInMap.(chan []call) <- nil
			}
			return newValue
		})
		if c.HeartBeat > 0 {
			ctx, cancel := context.WithTimeout(ctx, c.HeartBeat)
			defer cancel()
			select {
			case <-ctx.Done():
				responder <- emptyCall
			case result := <-responder:
				return result
			}
		}
	}
	return <-responder
}

func (c *Caller) end(ctx context.Context, results []returnValue) {
	id := c.ID(ctx)
	for _, rv := range results {
		if r, ok := c.results.Get(id); ok {
			if value := r.(*resultMap).GetAndDelete(rv.Index()); value != nil {
				value <- rv
			}
		}
	}
}

func (c *Caller) Invoke(id string, name string, args []interface{}, returnType ...reflect.Type) ([]interface{}, error) {
	return c.InvokeContext(context.Background(), id, name, args, returnType...)
}

func (c *Caller) InvokeContext(ctx context.Context, id string, name string, args []interface{}, returnType ...reflect.Type) ([]interface{}, error) {
	if args == nil {
		args = emptyArgs
	}
	index := int(atomic.AddInt32(&c.counter, 1) & 0x7fffffff)
	var calls *callCache
	if cc, ok := c.calls.Get(id); ok {
		calls = cc.(*callCache)
	} else {
		calls = new(callCache)
		if !c.calls.SetIfAbsent(id, calls) {
			cc, _ := c.calls.Get(id)
			calls = cc.(*callCache)
		}
	}
	calls.Append(newCall(index, name, args))
	var results *resultMap
	if rm, ok := c.results.Get(id); ok {
		results = rm.(*resultMap)
	} else {
		results = newResultMap()
		if !c.results.SetIfAbsent(id, results) {
			rm, _ := c.results.Get(id)
			results = rm.(*resultMap)
		}
	}
	result := make(chan returnValue, 1)
	results.Set(index, result)
	c.response(id)
	if c.Timeout > 0 {
		ctx, cancel := context.WithTimeout(ctx, c.HeartBeat)
		defer cancel()
		select {
		case <-ctx.Done():
			calls.Delete(index)
			results.Delete(index)
			return nil, core.ErrTimeout
		case result := <-result:
			return result.Value(returnType)
		}
	}
	return (<-result).Value(returnType)
}

func (c *Caller) UseService(remoteService interface{}, id string, namespace ...string) {
	ns := ""
	if len(namespace) > 0 {
		ns = namespace[0]
	}
	core.Proxy.Build(remoteService, invocation{caller: c, id: id, namespace: ns}.Invoke)
}

func (c *Caller) Exists(id string) bool {
	return c.onlines.Has(id)
}

func (c *Caller) IdList() []string {
	return c.onlines.Keys()
}

func (c *Caller) handler(ctx context.Context, name string, args []interface{}, next core.NextInvokeHandler) (result []interface{}, err error) {
	core.GetServiceContext(ctx).Items().Set("caller", c)
	return next(ctx, name, args)
}

func UseService(ctx context.Context, remoteService interface{}, namespace ...string) *Caller {
	caller := core.GetServiceContext(ctx).Items().GetInterface("caller").(*Caller)
	caller.UseService(remoteService, caller.ID(ctx), namespace...)
	return caller
}
<commit_msg>Fixed Timeout and HeartBeat<commit_after>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| rpc/plugins/reverse/caller.go                            |
|                                                          |
| LastModified: May 19, 2021                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package reverse

import (
	"context"
	"errors"
	"reflect"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hprose/hprose-golang/v3/io"
	"github.com/hprose/hprose-golang/v3/rpc/core"
	"github.com/modern-go/reflect2"
	cmap "github.com/orcaman/concurrent-map"
)

type call [3]interface{}

func newCall(index int, name string, args []interface{}) (c call) {
	c[0] = index
	c[1] = name
	c[2] = args
	return
}

func (c call) Value() (index int, name string, args []interface{}) {
	return c[0].(int), c[1].(string), c[2].([]interface{})
}

type callCache struct {
	c []call
	sync.Mutex
}

func (cc *callCache) Append(c call) {
	cc.Lock()
	defer cc.Unlock()
	cc.c = append(cc.c, c)
}

func (cc *callCache) Delete(index int) {
	cc.Lock()
	defer cc.Unlock()
	for i := 0; i < len(cc.c); i++ {
		if cc.c[i][0].(int) == index {
			cc.c = append(cc.c[:i], cc.c[i+1:]...)
			return
		}
	}
}

func (cc *callCache) Take() (calls []call) {
	cc.Lock()
	defer cc.Unlock()
	calls = cc.c
	cc.c = nil
	return
}

type returnValue [3]interface{}

func newReturnValue(index int, result interface{}, err string) (r returnValue) {
	r[0] = index
	r[1] = result
	r[2] = err
	return
}

func (r returnValue) Index() int {
	return r[0].(int)
}

func (r returnValue) Value(returnType []reflect.Type) ([]interface{}, error) {
	err := r[2].(string)
	if err != "" {
		return nil, errors.New(err)
	}
	n := len(returnType)
	switch n {
	case 0:
		return nil, nil
	case 1:
		if result, err := io.Convert(r[1], returnType[0]); err != nil {
			return nil, err
		} else {
			return []interface{}{result}, nil
		}
	default:
		results := make([]interface{}, n)
		values := r[1].([]interface{})
		count := len(values)
		for i := 0; i < n && i < count; i++ {
			if result, err := io.Convert(values[i], returnType[i]); err != nil {
				return nil, err
			} else {
				results[i] = result
			}
		}
		for i := count; i < n; i++ {
			t := reflect2.Type2(returnType[i])
			results[i] = t.Indirect(t.New())
		}
		return results, nil
	}
}

type resultMap struct {
	results map[int]chan returnValue
	sync.Mutex
}

func newResultMap() *resultMap {
	return &resultMap{
		results: make(map[int]chan returnValue),
	}
}

func (m *resultMap) GetAndDelete(index int) chan returnValue {
	m.Lock()
	defer m.Unlock()
	if result, ok := m.results[index]; ok {
		delete(m.results, index)
		return result
	}
	return nil
}

func (m *resultMap) Delete(index int) {
	m.Lock()
	defer m.Unlock()
	delete(m.results, index)
}

func (m *resultMap) Set(index int, result chan returnValue) {
	m.Lock()
	defer m.Unlock()
	m.results[index] = result
}

var (
	emptyArgs = make([]interface{}, 0)
	emptyCall = make([]call, 0)
)

type Caller struct {
	*core.Service
	IdleTimeout time.Duration
	Timeout     time.Duration
	HeartBeat   time.Duration
	calls       cmap.ConcurrentMap
	results     cmap.ConcurrentMap
	responders  cmap.ConcurrentMap
	onlines     cmap.ConcurrentMap
	counter     int32
}

func NewCaller(service *core.Service) *Caller {
	caller := &Caller{
		Service:     service,
		IdleTimeout: time.Minute * 2,
		Timeout:     time.Second * 30,
		HeartBeat:   time.Second * 3,
		calls:       cmap.New(),
		results:     cmap.New(),
		responders:  cmap.New(),
		onlines:     cmap.New(),
	}
	service.Use(caller.handler).
		AddFunction(caller.close, "!!").
		AddFunction(caller.begin, "!").
		AddFunction(caller.end, "=")
	return caller
}

func (c *Caller) ID(ctx context.Context) (id string) {
	if id = core.GetServiceContext(ctx).RequestHeaders().GetString("id"); id == "" {
		panic("client unique id not found")
	}
	return
}

func (c *Caller) send(id string, responder chan []call) bool {
	if calls, ok := c.calls.Get(id); ok {
		calls := calls.(*callCache).Take()
		if len(calls) == 0 {
			return false
		}
		responder <- calls
		return true
	}
	return false
}

func (c *Caller) response(id string) {
	if responder, ok := c.responders.Pop(id); ok {
		responder := responder.(chan []call)
		if !c.send(id, responder) {
			if c.responders.SetIfAbsent(id, responder) {
				responder <- nil
			}
		}
	}
}

func (c *Caller) stop(ctx context.Context) string {
	id := c.ID(ctx)
	if responder, ok := c.responders.Pop(id); ok {
		responder.(chan []call) <- nil
	}
	return id
}

func (c *Caller) close(ctx context.Context) {
	id := c.stop(ctx)
	c.onlines.Remove(id)
}

func (c *Caller) begin(ctx context.Context) []call {
	id := c.stop(ctx)
	online := make(chan bool, 1)
	c.onlines.Upsert(id, online, func(exist bool, valueInMap, newValue interface{}) interface{} {
		if exist {
			if online, ok := valueInMap.(chan bool); ok {
				online <- true
			}
		}
		return newValue
	})
	if c.HeartBeat > 0 {
		defer func() {
			go func() {
				ctx, cancel := context.WithTimeout(context.Background(), c.HeartBeat)
				defer cancel()
				select {
				case <-ctx.Done():
					c.onlines.Remove(id)
				case <-online:
				}
			}()
		}()
	}
	responder := make(chan []call, 1)
	if !c.send(id, responder) {
		c.responders.Upsert(id, responder, func(exist bool, valueInMap interface{}, newValue interface{}) interface{} {
			if exist {
				valueInMap.(chan []call) <- nil
			}
			return newValue
		})
		if c.IdleTimeout > 0 {
			ctx, cancel := context.WithTimeout(ctx, c.IdleTimeout)
			defer cancel()
			select {
			case <-ctx.Done():
				responder <- emptyCall
			case result := <-responder:
				return result
			}
		}
	}
	return <-responder
}

func (c *Caller) end(ctx context.Context, results []returnValue) {
	id := c.ID(ctx)
	for _, rv := range results {
		if r, ok := c.results.Get(id); ok {
			if value := r.(*resultMap).GetAndDelete(rv.Index()); value != nil {
				value <- rv
			}
		}
	}
}

func (c *Caller) Invoke(id string, name string, args []interface{}, returnType ...reflect.Type) ([]interface{}, error) {
	return c.InvokeContext(context.Background(), id, name, args, returnType...)
}

func (c *Caller) InvokeContext(ctx context.Context, id string, name string, args []interface{}, returnType ...reflect.Type) ([]interface{}, error) {
	if args == nil {
		args = emptyArgs
	}
	index := int(atomic.AddInt32(&c.counter, 1) & 0x7fffffff)
	var calls *callCache
	if cc, ok := c.calls.Get(id); ok {
		calls = cc.(*callCache)
	} else {
		calls = new(callCache)
		if !c.calls.SetIfAbsent(id, calls) {
			cc, _ := c.calls.Get(id)
			calls = cc.(*callCache)
		}
	}
	calls.Append(newCall(index, name, args))
	var results *resultMap
	if rm, ok := c.results.Get(id); ok {
		results = rm.(*resultMap)
	} else {
		results = newResultMap()
		if !c.results.SetIfAbsent(id, results) {
			rm, _ := c.results.Get(id)
			results = rm.(*resultMap)
		}
	}
	result := make(chan returnValue, 1)
	results.Set(index, result)
	c.response(id)
	if c.Timeout > 0 {
		ctx, cancel := context.WithTimeout(ctx, c.Timeout)
		defer cancel()
		select {
		case <-ctx.Done():
			calls.Delete(index)
			results.Delete(index)
			return nil, core.ErrTimeout
		case result := <-result:
			return result.Value(returnType)
		}
	}
	return (<-result).Value(returnType)
}

func (c *Caller) UseService(remoteService interface{}, id string, namespace ...string) {
	ns := ""
	if len(namespace) > 0 {
		ns = namespace[0]
	}
	core.Proxy.Build(remoteService, invocation{caller: c, id: id, namespace: ns}.Invoke)
}

func (c *Caller) Exists(id string) bool {
	return c.onlines.Has(id)
}

func (c *Caller) IdList() []string {
	return c.onlines.Keys()
}

func (c *Caller) handler(ctx context.Context, name string, args []interface{}, next core.NextInvokeHandler) (result []interface{}, err error) {
	core.GetServiceContext(ctx).Items().Set("caller", c)
	return next(ctx, name, args)
}

func UseService(ctx context.Context, remoteService interface{}, namespace ...string) *Caller {
	caller := core.GetServiceContext(ctx).Items().GetInterface("caller").(*Caller)
	caller.UseService(remoteService, caller.ID(ctx), namespace...)
	return caller
}
<|endoftext|>


<commit_before>package shell

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandFsMetaSave{})
}

type commandFsMetaSave struct {
}

func (c *commandFsMetaSave) Name() string {
	return "fs.meta.save"
}

func (c *commandFsMetaSave) Help() string {
	return `save all directory and file meta data to a local file for metadata backup.

	fs.meta.save /               # save from the root
	fs.meta.save -v -o t.meta /  # save from the root, output to t.meta file.
	fs.meta.save /path/to/save   # save from the directory /path/to/save
	fs.meta.save .               # save from current directory
	fs.meta.save                 # save from current directory

	The meta data will be saved into a local <filer_host>-<port>-<time>.meta file.
	These meta data can be later loaded by fs.meta.load command,

`
}

func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
	outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file")
	isObfuscate := fsMetaSaveCommand.Bool("obfuscate", false, "obfuscate the file names")
	// chunksFileName := fsMetaSaveCommand.String("chunks", "", "output all the chunks to this file")
	if err = fsMetaSaveCommand.Parse(args); err != nil {
		return nil
	}

	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
	if parseErr != nil {
		return parseErr
	}

	fileName := *outputFileName
	if fileName == "" {
		t := time.Now()
		fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
			commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	}

	dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if openErr != nil {
		return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
	}
	defer dst.Close()

	var cipherKey util.CipherKey
	if *isObfuscate {
		cipherKey = util.GenCipherKey()
	}

	err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(outputChan chan interface{}) {
		sizeBuf := make([]byte, 4)
		for item := range outputChan {
			b := item.([]byte)
			util.Uint32toBytes(sizeBuf, uint32(len(b)))
			dst.Write(sizeBuf)
			dst.Write(b)
		}
	}, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
		if !entry.Entry.IsDirectory {
			ext := filepath.Ext(entry.Entry.Name)
			if encrypted, encErr := util.Encrypt([]byte(entry.Entry.Name), cipherKey); encErr == nil {
				entry.Entry.Name = util.Base64Encode(encrypted)[:len(entry.Entry.Name)] + ext
				entry.Entry.Name = strings.ReplaceAll(entry.Entry.Name, "/", "x")
			}
		}
		bytes, err := proto.Marshal(entry)
		if err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		outputChan <- bytes
		return nil
	})

	if err == nil {
		fmt.Fprintf(writer, "meta data for http://%s:%d%s is saved to %s\n", commandEnv.option.FilerHost, commandEnv.option.FilerPort, path, fileName)
	}

	return err

}

func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, saveFn func(outputChan chan interface{}), genFn func(entry *filer_pb.FullEntry, outputChan chan interface{}) error) error {

	var wg sync.WaitGroup
	wg.Add(1)
	outputChan := make(chan interface{}, 1024)
	go func() {
		saveFn(outputChan)
		wg.Done()
	}()

	var dirCount, fileCount uint64

	err := filer_pb.TraverseBfs(filerClient, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {

		protoMessage := &filer_pb.FullEntry{
			Dir:   string(parentPath),
			Entry: entry,
		}

		if err := genFn(protoMessage, outputChan); err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		if entry.IsDirectory {
			atomic.AddUint64(&dirCount, 1)
		} else {
			atomic.AddUint64(&fileCount, 1)
		}

		if verbose {
			println(parentPath.Child(entry.Name))
		}

	})

	close(outputChan)

	wg.Wait()

	if err == nil && writer != nil {
		fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
	}
	return err
}
<commit_msg>minor<commit_after>package shell

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandFsMetaSave{})
}

type commandFsMetaSave struct {
}

func (c *commandFsMetaSave) Name() string {
	return "fs.meta.save"
}

func (c *commandFsMetaSave) Help() string {
	return `save all directory and file meta data to a local file for metadata backup.

	fs.meta.save /               # save from the root
	fs.meta.save -v -o t.meta /  # save from the root, output to t.meta file.
	fs.meta.save /path/to/save   # save from the directory /path/to/save
	fs.meta.save .               # save from current directory
	fs.meta.save                 # save from current directory

	The meta data will be saved into a local <filer_host>-<port>-<time>.meta file.
	These meta data can be later loaded by fs.meta.load command

`
}

func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
	outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file")
	isObfuscate := fsMetaSaveCommand.Bool("obfuscate", false, "obfuscate the file names")
	// chunksFileName := fsMetaSaveCommand.String("chunks", "", "output all the chunks to this file")
	if err = fsMetaSaveCommand.Parse(args); err != nil {
		return nil
	}

	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
	if parseErr != nil {
		return parseErr
	}

	fileName := *outputFileName
	if fileName == "" {
		t := time.Now()
		fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
			commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	}

	dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if openErr != nil {
		return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
	}
	defer dst.Close()

	var cipherKey util.CipherKey
	if *isObfuscate {
		cipherKey = util.GenCipherKey()
	}

	err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(outputChan chan interface{}) {
		sizeBuf := make([]byte, 4)
		for item := range outputChan {
			b := item.([]byte)
			util.Uint32toBytes(sizeBuf, uint32(len(b)))
			dst.Write(sizeBuf)
			dst.Write(b)
		}
	}, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
		if !entry.Entry.IsDirectory {
			ext := filepath.Ext(entry.Entry.Name)
			if encrypted, encErr := util.Encrypt([]byte(entry.Entry.Name), cipherKey); encErr == nil {
				entry.Entry.Name = util.Base64Encode(encrypted)[:len(entry.Entry.Name)] + ext
				entry.Entry.Name = strings.ReplaceAll(entry.Entry.Name, "/", "x")
			}
		}
		bytes, err := proto.Marshal(entry)
		if err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		outputChan <- bytes
		return nil
	})

	if err == nil {
		fmt.Fprintf(writer, "meta data for http://%s:%d%s is saved to %s\n", commandEnv.option.FilerHost, commandEnv.option.FilerPort, path, fileName)
	}

	return err

}

func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, saveFn func(outputChan chan interface{}), genFn func(entry *filer_pb.FullEntry, outputChan chan interface{}) error) error {

	var wg sync.WaitGroup
	wg.Add(1)
	outputChan := make(chan interface{}, 1024)
	go func() {
		saveFn(outputChan)
		wg.Done()
	}()

	var dirCount, fileCount uint64

	err := filer_pb.TraverseBfs(filerClient, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {

		protoMessage := &filer_pb.FullEntry{
			Dir:   string(parentPath),
			Entry: entry,
		}

		if err := genFn(protoMessage, outputChan); err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		if entry.IsDirectory {
			atomic.AddUint64(&dirCount, 1)
		} else {
			atomic.AddUint64(&fileCount, 1)
		}

		if verbose {
			println(parentPath.Child(entry.Name))
		}

	})

	close(outputChan)

	wg.Wait()

	if err == nil && writer != nil {
		fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
	}
	return err
}
<|endoftext|>


<commit_before>/*
Package html2data - extract data from HTML via CSS selectors

Install package and command line utility:

	go get -u github.com/msoap/html2data/cmd/html2data

Install package only:

	go get -u github.com/msoap/html2data

Allowed pseudo-selectors:

:attr(attr_name) - for getting attributes instead text

:html - for getting HTML instead text

:get(N) - get n-th element from list

Command line utility:

	html2data URL "css selector"
	html2data file.html "css selector"
	cat file.html | html2data "css selector"

*/
package html2data

import (
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
)

// docOrSelection - for exec .Find
type docOrSelection interface {
	Find(string) *goquery.Selection
}

// Doc - html document for parse
type Doc struct {
	doc docOrSelection
	Err error
}

// CSSSelector - selector with settings
type CSSSelector struct {
	selector string
	attrName string
	getHTML  bool
	getNth   int
}

// Cfg - config for GetData* methods
type Cfg struct {
	DontTrimSpaces bool // get text as is, by default trim spaces
}

// getDataFromDocOrSelection - extract data by CSS-selectors from goquery.Selection or goquery.Doc
func (doc Doc) getDataFromDocOrSelection(docOrSelection docOrSelection, selectors map[string]string, config Cfg) (result map[string][]string, err error) {
	if doc.Err != nil {
		return result, fmt.Errorf("parse document error: %s", doc.Err)
	}
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()

	result = map[string][]string{}
	for name, selectorRaw := range selectors {
		selector := parseSelector(selectorRaw)

		texts := []string{}
		docOrSelection.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
			if selector.getNth > 0 && selector.getNth != i+1 {
				return
			}

			foundText := ""
			switch {
			case selector.attrName != "":
				foundText = selection.AttrOr(selector.attrName, "")
			case selector.getHTML:
				foundText, err = selection.Html()
				if err != nil {
					return
				}
			default:
				foundText = selection.Text()
			}

			if !config.DontTrimSpaces {
				foundText = strings.TrimSpace(foundText)
			}
			texts = append(texts, foundText)
		})
		result[name] = texts
	}

	return result, err
}

// parseSelector - parse pseudo-selectors:
// :attr(href) - for getting attribute instead text node
func parseSelector(inputSelector string) (outSelector CSSSelector) {
	htmlAttrRe := regexp.MustCompile(`^\s*(\w+)\s*(?:\(\s*(\w+)\s*\))?\s*$`)

	parts := strings.Split(inputSelector, ":")
	outSelector.selector, parts = parts[0], parts[1:]
	for _, part := range parts {
		reParts := htmlAttrRe.FindStringSubmatch(part)
		switch {
		case len(reParts) == 3 && reParts[1] == "attr":
			outSelector.attrName = reParts[2]
		case len(reParts) == 3 && reParts[1] == "html":
			outSelector.getHTML = true
		case len(reParts) == 3 && reParts[1] == "get":
			outSelector.getNth, _ = strconv.Atoi(reParts[2])
		default:
			outSelector.selector += ":" + part
		}
	}

	return outSelector
}

// getConfig - get first config element from list
func getConfig(configs []Cfg) Cfg {
	switch {
	case len(configs) == 0:
		return Cfg{}
	case len(configs) == 1:
		return configs[0]
	default:
		panic("[]Cfg length must be equal 0 or 1")
	}
}

// GetData - extract data by CSS-selectors
//  texts, err := doc.GetData(map[string]string{"h1": "h1"})
func (doc Doc) GetData(selectors map[string]string, configs ...Cfg) (result map[string][]string, err error) {
	result, err = doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	return result, err
}

// GetDataFirst - extract data by CSS-selectors, get first entry for each selector or ""
//  texts, err := doc.GetDataFirst(map[string]string{"h1": "h1"})
func (doc Doc) GetDataFirst(selectors map[string]string, configs ...Cfg) (result map[string]string, err error) {
	resultRaw, err := doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	if err != nil {
		return result, err
	}

	result = map[string]string{}
	for key := range resultRaw {
		if len(resultRaw[key]) > 0 {
			result[key] = resultRaw[key][0]
		} else {
			result[key] = ""
		}
	}

	return result, err
}

// GetDataNested - extract nested data by CSS-selectors from another CSS-selector
//  texts, err := doc.GetDataNested("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNested(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string][]string, err error) {
	selector := parseSelector(selectorRaw)
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = []map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()
	result = []map[string][]string{}

	doc.doc.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
		if selector.getNth > 0 && selector.getNth != i+1 {
			return
		}

		nestedResult, nestedErr := doc.getDataFromDocOrSelection(selection, nestedSelectors, getConfig(configs))
		if nestedErr != nil {
			err = nestedErr
			return
		}

		result = append(result, nestedResult)
	})

	return result, err
}

// GetDataNestedFirst - extract nested data by CSS-selectors from another CSS-selector
// get first entry for each selector or ""
//  texts, err := doc.GetDataNestedFirst("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNestedFirst(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string]string, err error) {
	resultRaw, err := doc.GetDataNested(selectorRaw, nestedSelectors, configs...)
	if err != nil {
		return result, err
	}

	result = []map[string]string{}
	for i, resultRawPart := range resultRaw {
		result = append(result, map[string]string{})
		for key := range resultRawPart {
			if len(resultRawPart[key]) > 0 {
				result[i][key] = resultRawPart[key][0]
			} else {
				result[i][key] = ""
			}
		}
	}

	return result, err
}

// GetDataSingle - extract data by one CSS-selector
//  title, err := doc.GetDataSingle("title")
func (doc Doc) GetDataSingle(selector string, configs ...Cfg) (result string, err error) {
	texts, err := doc.GetData(map[string]string{"single": selector}, getConfig(configs))
	if err != nil {
		return result, err
	}

	if textOne, ok := texts["single"]; ok && len(textOne) > 0 {
		result = textOne[0]
	}

	return result, err
}

// FromReader - get doc from io.Reader
func FromReader(reader io.Reader) Doc {
	doc, err := goquery.NewDocumentFromReader(reader)
	return Doc{doc, err}
}

// FromFile - get doc from file
func FromFile(fileName string) Doc {
	fileReader, err := os.Open(fileName)
	if err != nil {
		return Doc{Err: err}
	}

	doc := FromReader(fileReader)
	err = fileReader.Close()
	if err != nil {
		return Doc{Err: err}
	}

	return doc
}

// URLCfg - config for FromURL()
type URLCfg struct {
	UA                string // custom user-agent
	TimeOut           int    // timeout in seconds
	DontDetectCharset bool   // dont autoconvert to UTF8
}

// FromURL - get doc from URL
//
//  FromURL("https://url")
//  FromURL("https://url", URLCfg{UA: "Custom UA 1.0", TimeOut: 10})
func FromURL(URL string, config ...URLCfg) Doc {
	ua, timeout, dontDetectCharset := "", 0, false
	if len(config) == 1 {
		ua = config[0].UA
		timeout = config[0].TimeOut
		dontDetectCharset = config[0].DontDetectCharset
	} else if len(config) > 1 {
		panic("FromURL(): only one config argument allowed")
	}

	htmlReader, err := getHTMLPage(URL, ua, timeout, dontDetectCharset)
	if err != nil {
		return Doc{Err: err}
	}

	return FromReader(htmlReader)
}

// getHTMLPage - get html by http(s) as http.Response
func getHTMLPage(url string, ua string, timeout int, dontDetectCharset bool) (htmlReader io.Reader, err error) {
	cookie, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookie,
		Timeout: time.Duration(timeout) * time.Second,
	}

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return htmlReader, err
	}

	if ua != "" {
		request.Header.Set("User-Agent", ua)
	}

	response, err := client.Do(request)
	if err != nil {
		return htmlReader, err
	}

	if contentType := response.Header.Get("Content-Type"); contentType != "" && !dontDetectCharset {
		htmlReader, err = charset.NewReader(response.Body, contentType)
		if err != nil {
			return htmlReader, err
		}
	} else {
		return response.Body, nil
	}

	return htmlReader, nil
}
<commit_msg>Revert 751a88d1a032b9f4b0a4c7b299a5698b7fd1d616<commit_after>/*
Package html2data - extract data from HTML via CSS selectors

Install package and command line utility:

	go get -u github.com/msoap/html2data/cmd/html2data

Install package only:

	go get -u github.com/msoap/html2data

Allowed pseudo-selectors:

:attr(attr_name) - for getting attributes instead text

:html - for getting HTML instead text

:get(N) - get n-th element from list

Command line utility:

	html2data URL "css selector"
	html2data file.html "css selector"
	cat file.html | html2data "css selector"

*/
package html2data

import (
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
)

// docOrSelection - for exec .Find
type docOrSelection interface {
	Find(string) *goquery.Selection
}

// Doc - html document for parse
type Doc struct {
	doc docOrSelection
	Err error
}

// CSSSelector - selector with settings
type CSSSelector struct {
	selector string
	attrName string
	getHTML  bool
	getNth   int
}

// Cfg - config for GetData* methods
type Cfg struct {
	DontTrimSpaces bool // get text as is, by default trim spaces
}

// getDataFromDocOrSelection - extract data by CSS-selectors from goquery.Selection or goquery.Doc
func (doc Doc) getDataFromDocOrSelection(docOrSelection docOrSelection, selectors map[string]string, config Cfg) (result map[string][]string, err error) {
	if doc.Err != nil {
		return result, fmt.Errorf("parse document error: %s", doc.Err)
	}
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()

	result = map[string][]string{}
	for name, selectorRaw := range selectors {
		selector := parseSelector(selectorRaw)

		texts := []string{}
		docOrSelection.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
			if selector.getNth > 0 && selector.getNth != i+1 {
				return
			}

			foundText := ""
			switch {
			case selector.attrName != "":
				foundText = selection.AttrOr(selector.attrName, "")
			case selector.getHTML:
				foundText, err = selection.Html()
				if err != nil {
					return
				}
			default:
				foundText = selection.Text()
			}

			if !config.DontTrimSpaces {
				foundText = strings.TrimSpace(foundText)
			}
			texts = append(texts, foundText)
		})
		result[name] = texts
	}

	return result, err
}

// parseSelector - parse pseudo-selectors:
// :attr(href) - for getting attribute instead text node
func parseSelector(inputSelector string) (outSelector CSSSelector) {
	htmlAttrRe := regexp.MustCompile(`^\s*(\w+)\s*(?:\(\s*(\w+)\s*\))?\s*$`)

	parts := strings.Split(inputSelector, ":")
	outSelector.selector, parts = parts[0], parts[1:]
	for _, part := range parts {
		reParts := htmlAttrRe.FindStringSubmatch(part)
		switch {
		case len(reParts) == 3 && reParts[1] == "attr":
			outSelector.attrName = reParts[2]
		case len(reParts) == 3 && reParts[1] == "html":
			outSelector.getHTML = true
		case len(reParts) == 3 && reParts[1] == "get":
			outSelector.getNth, _ = strconv.Atoi(reParts[2])
		default:
			outSelector.selector += ":" + part
		}
	}

	return outSelector
}

// getConfig - get first config element from list
func getConfig(configs []Cfg) Cfg {
	switch {
	case len(configs) == 0:
		return Cfg{}
	case len(configs) == 1:
		return configs[0]
	default:
		panic("[]Cfg length must be equal 0 or 1")
	}
}

// GetData - extract data by CSS-selectors
//  texts, err := doc.GetData(map[string]string{"h1": "h1"})
func (doc Doc) GetData(selectors map[string]string, configs ...Cfg) (result map[string][]string, err error) {
	result, err = doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	return result, err
}

// GetDataFirst - extract data by CSS-selectors, get first entry for each selector or ""
//  texts, err := doc.GetDataFirst(map[string]string{"h1": "h1"})
func (doc Doc) GetDataFirst(selectors map[string]string, configs ...Cfg) (result map[string]string, err error) {
	resultRaw, err := doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	if err != nil {
		return result, err
	}

	result = map[string]string{}
	for key := range resultRaw {
		if len(resultRaw[key]) > 0 {
			result[key] = resultRaw[key][0]
		} else {
			result[key] = ""
		}
	}

	return result, err
}

// GetDataNested - extract nested data by CSS-selectors from another CSS-selector
//  texts, err := doc.GetDataNested("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNested(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string][]string, err error) {
	selector := parseSelector(selectorRaw)
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = []map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()
	result = []map[string][]string{}

	doc.doc.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
		if selector.getNth > 0 && selector.getNth != i+1 {
			return
		}

		nestedResult, nestedErr := doc.getDataFromDocOrSelection(selection, nestedSelectors, getConfig(configs))
		if nestedErr != nil {
			err = nestedErr
			return
		}

		result = append(result, nestedResult)
	})

	return result, err
}

// GetDataNestedFirst - extract nested data by CSS-selectors from another CSS-selector
// get first entry for each selector or ""
//  texts, err := doc.GetDataNestedFirst("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNestedFirst(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string]string, err error) {
	resultRaw, err := doc.GetDataNested(selectorRaw, nestedSelectors, configs...)
	if err != nil {
		return result, err
	}

	result = []map[string]string{}
	for i, resultRawPart := range resultRaw {
		result = append(result, map[string]string{})
		for key := range resultRawPart {
			if len(resultRawPart[key]) > 0 {
				result[i][key] = resultRawPart[key][0]
			} else {
				result[i][key] = ""
			}
		}
	}

	return result, err
}

// GetDataSingle - extract data by one CSS-selector
//  title, err := doc.GetDataSingle("title")
func (doc Doc) GetDataSingle(selector string, configs ...Cfg) (result string, err error) {
	texts, err := doc.GetData(map[string]string{"single": selector}, getConfig(configs))
	if err != nil {
		return result, err
	}

	if textOne, ok := texts["single"]; ok && len(textOne) > 0 {
		result = textOne[0]
	}

	return result, err
}

// FromReader - get doc from io.Reader
func FromReader(reader io.Reader) Doc {
	doc, err := goquery.NewDocumentFromReader(reader)
	return Doc{doc, err}
}

// FromFile - get doc from file
func FromFile(fileName string) Doc {
	fileReader, err := os.Open(fileName)
	if err != nil {
		return Doc{Err: err}
	}

	doc := FromReader(fileReader)
	err = fileReader.Close()
	if err != nil {
		return Doc{Err: err}
	}

	return doc
}

// URLCfg - config for FromURL()
type URLCfg struct {
	UA                string // custom user-agent
	TimeOut           int    // timeout in seconds
	DontDetectCharset bool   // dont autoconvert to UTF8
}

// FromURL - get doc from URL
//
//  FromURL("https://url")
//  FromURL("https://url", URLCfg{UA: "Custom UA 1.0", TimeOut: 10})
func FromURL(URL string, config ...URLCfg) Doc {
	ua, timeout, dontDetectCharset := "", 0, false
	if len(config) == 1 {
		ua = config[0].UA
		timeout = config[0].TimeOut
		dontDetectCharset = config[0].DontDetectCharset
	} else if len(config) > 1 {
		panic("FromURL(): only one config argument allowed")
	}

	htmlReader, err := getHTMLPage(URL, ua, timeout, dontDetectCharset)
	if err != nil {
		return Doc{Err: err}
	}

	return FromReader(htmlReader)
}

// getHTMLPage - get html by http(s) as http.Response
func getHTMLPage(url string, ua string, timeout int, dontDetectCharset bool) (htmlReader io.Reader, err error) {
	cookie, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookie,
		Timeout: time.Duration(time.Duration(timeout) * time.Second),
	}

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return htmlReader, err
	}

	if ua != "" {
		request.Header.Set("User-Agent", ua)
	}

	response, err := client.Do(request)
	if err != nil {
		return htmlReader, err
	}

	if contentType := response.Header.Get("Content-Type"); contentType != "" && !dontDetectCharset {
		htmlReader, err = charset.NewReader(response.Body, contentType)
		if err != nil {
			return htmlReader, err
		}
	} else {
		return response.Body, nil
	}

	return htmlReader, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/SocialCodeInc/go-gelf/gelf"
	"io"
	"bufio"
	"strconv"
	"log"
	"os/exec"
	"time"
	"os"
	"errors"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor string			`json:"__CURSOR"`
	Realtime_timestamp string	`json:"__REALTIME_TIMESTAMP"`
	Monotonic_timestamp string	`json:"__MONOTONIC_TIMESTAMP"`
	Boot_id string			`json:"_BOOT_ID"`
	Transport string		`json:"_TRANSPORT"`
	Priority string			`json:"PRIORITY"`
	Syslog_facility string		`json:"SYSLOG_FACILITY"`
	Syslog_identifier string	`json:"SYSLOG_IDENTIFIER"`
	Message string			`json:"MESSAGE"`
	Pid string			`json:"_PID"`
	Uid string			`json:"_UID"`
	Gid string			`json:"_GID"`
	Comm string			`json:"_COMM"`
	Exe string			`json:"_EXE"`
	Cmdline string			`json:"_CMDLINE"`
	Systemd_cgroup string		`json:"_SYSTEMD_CGROUP"`
	Systemd_session string		`json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid string	`json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit string		`json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp string`json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id string		`json:"_MACHINE_ID"`
	Hostname string			`json:"_HOSTNAME"`
}

func (this *SystemdJournalEntry) toGelf() (*gelf.Message, error) {
	timestamp, err := strconv.ParseInt(this.Realtime_timestamp, 10, 64)
	if err != nil {
		return nil, errors.New(this.Cursor+" - error in timestamp: "+ err.Error())
	}

	level, err := strconv.ParseInt(this.Priority, 10, 32)
	if err != nil {
		return nil, errors.New(this.Cursor+" - error in priority: "+ err.Error())
	}

	return &gelf.Message{
		Version:	"1.0",
		Host:		this.Hostname,
		Short:		this.Message,
		TimeUnix:	timestamp / 1000 / 1000,
		Level:		int32(level),
		Facility:	this.Systemd_unit,
		Extra:		map[string] interface {}{
			"Boot_id":	this.Boot_id,
			"Pid":		this.Pid,
			"Uid":		this.Uid,
		},
	}, nil
}

func main() {
	if len(os.Args) < 3 {
		log.Fatal("Pass server:12201 as first argument and append journalctl parameters to use")
	}

	serverAddr := os.Args[1]
	journalArgs := os.Args[1:]
	journalArgs[0] = "--output=json"

	cmd := exec.Command("journalctl", journalArgs...)

	gelfWriter, err := gelf.NewWriter(serverAddr)
	if err != nil {
		log.Fatalf("While connecting to Graylog server: %s", err)
	}

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)

	// Larger buffer for systemd's inline coredumps which are typically ~ 14Kb
	r := bufio.NewReaderSize(stdout, 16484)
	cmd.Start()

	for line, _, err := r.ReadLine(); err != io.EOF; line, _, err = r.ReadLine() {
		if err != nil {
			break
		}

		entry := new(SystemdJournalEntry);
		err = json.Unmarshal(line, &entry)

		if err != nil {
			log.Printf("Could not parse line, skipping: %s", line)
			continue
		}

		message, err := entry.toGelf();

		if err != nil {
			log.Print(err)
			continue
		} else {
			go gelfWriter.WriteMessage(message)

			// Prevent saturation and throttling
			time.Sleep(2 * time.Millisecond)
		}
	}

	cmd.Wait()
}
<commit_msg>Syslog_identifier is filled more often<commit_after>package main

import (
	"encoding/json"
	"github.com/SocialCodeInc/go-gelf/gelf"
	"io"
	"bufio"
	"strconv"
	"log"
	"os/exec"
	"time"
	"os"
	"errors"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor string			`json:"__CURSOR"`
	Realtime_timestamp string	`json:"__REALTIME_TIMESTAMP"`
	Monotonic_timestamp string	`json:"__MONOTONIC_TIMESTAMP"`
	Boot_id string			`json:"_BOOT_ID"`
	Transport string		`json:"_TRANSPORT"`
	Priority string			`json:"PRIORITY"`
	Syslog_facility string		`json:"SYSLOG_FACILITY"`
	Syslog_identifier string	`json:"SYSLOG_IDENTIFIER"`
	Message string			`json:"MESSAGE"`
	Pid string			`json:"_PID"`
	Uid string			`json:"_UID"`
	Gid string			`json:"_GID"`
	Comm string			`json:"_COMM"`
	Exe string			`json:"_EXE"`
	Cmdline string			`json:"_CMDLINE"`
	Systemd_cgroup string		`json:"_SYSTEMD_CGROUP"`
	Systemd_session string		`json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid string	`json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit string		`json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp string`json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id string		`json:"_MACHINE_ID"`
	Hostname string			`json:"_HOSTNAME"`
}

func (this *SystemdJournalEntry) toGelf() (*gelf.Message, error) {
	timestamp, err := strconv.ParseInt(this.Realtime_timestamp, 10, 64)
	if err != nil {
		return nil, errors.New(this.Cursor+" - error in timestamp: "+ err.Error())
	}

	level, err := strconv.ParseInt(this.Priority, 10, 32)
	if err != nil {
		return nil, errors.New(this.Cursor+" - error in priority: "+ err.Error())
	}

	return &gelf.Message{
		Version:	"1.0",
		Host:		this.Hostname,
		Short:		this.Message,
		TimeUnix:	timestamp / 1000 / 1000,
		Level:		int32(level),
		Facility:	this.Syslog_identifier,
		Extra:		map[string] interface {}{
			"Boot_id":	this.Boot_id,
			"Pid":		this.Pid,
			"Uid":		this.Uid,
		},
	}, nil
}

func main() {
	if len(os.Args) < 3 {
		log.Fatal("Pass server:12201 as first argument and append journalctl parameters to use")
	}

	serverAddr := os.Args[1]
	journalArgs := os.Args[1:]
	journalArgs[0] = "--output=json"

	cmd := exec.Command("journalctl", journalArgs...)

	gelfWriter, err := gelf.NewWriter(serverAddr)
	if err != nil {
		log.Fatalf("While connecting to Graylog server: %s", err)
	}

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)

	// Larger buffer for systemd's inline coredumps which are typically ~ 14Kb
	r := bufio.NewReaderSize(stdout, 16484)
	cmd.Start()

	for line, _, err := r.ReadLine(); err != io.EOF; line, _, err = r.ReadLine() {
		if err != nil {
			break
		}

		entry := new(SystemdJournalEntry);
		err = json.Unmarshal(line, &entry)

		if err != nil {
			log.Printf("Could not parse line, skipping: %s", line)
			continue
		}

		message, err := entry.toGelf();

		if err != nil {
			log.Print(err)
			continue
		} else {
			go gelfWriter.WriteMessage(message)

			// Prevent saturation and throttling
			time.Sleep(2 * time.Millisecond)
		}
	}

	cmd.Wait()
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Space Monkey, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/spacemonkeygo/monkit/v3"
)

// see: https://github.com/w3c/trace-context/blob/main/spec/20-http_request_header_format.md
const (
	traceSampled      = byte(1)
	traceParentHeader = "traceparent"
	traceStateHeader  = "tracestate"

	// orphanSampling is a special k,v which can be added to the vendor specific tracestate header.
	// it can turn on trace sampling on remote even without propagataing the parent trace
	// (traceparent must not contain zero IDs)
	// useful when you use curl (no client side tracing), and would like to get traces from the server.
	orphanSampling = "sampled=true"
)

// TraceInfo is a structure representing an incoming RPC request. Every field
// is optional.
type TraceInfo struct {
	TraceId  *int64
	ParentId *int64
	Sampled  bool
}

// HeaderGetter is an interface that http.Header matches for RequestFromHeader
type HeaderGetter interface {
	Get(string) string
}

// HeaderSetter is an interface that http.Header matches for TraceInfo.SetHeader
type HeaderSetter interface {
	Set(string, string)
}

// TraceInfoFromHeader will create a TraceInfo object given an http.Header or
// anything that matches the HeaderGetter interface.
func TraceInfoFromHeader(header HeaderGetter) (rv TraceInfo) {
	traceParent := header.Get(traceParentHeader)
	traceState := header.Get(traceStateHeader)

	if traceParent != "" {
		parts := strings.Split(traceParent, "-")
		if len(parts) != 4 {
			return rv
		}
		version, err := hexToUint64(parts[0])
		if err != nil || version != 0 {
			return rv
		}
		traceID, err := hexToUint64(parts[1])
		if err != nil {
			return rv
		}
		parentID, err := hexToUint64(parts[2])
		if err != nil {
			return rv
		}
		flags, err := hexToUint64(parts[3])
		if err != nil {
			return rv
		}

		return TraceInfo{
			TraceId:  &traceID,
			ParentId: &parentID,
			Sampled:  (byte(flags) & traceSampled) == traceSampled,
		}
	}

	// trace parent is not set, but tracing can be turned on by a traceState
	if strings.Contains(traceState, "sampled=true") {
		return TraceInfo{
			Sampled: true,
		}
	}
	return rv
}

func ref(v int64) *int64 {
	return &v
}

func TraceInfoFromSpan(s *monkit.Span, sampledf func(trace *monkit.Trace) bool) TraceInfo {
	trace := s.Trace()

	sampled := sampledf(trace)

	if !sampled {
		return TraceInfo{Sampled: sampled}
	}

	req := TraceInfo{
		TraceId:  ref(trace.Id()),
		ParentId: ref(s.Id()),
		Sampled:  sampled,
	}
	if parentID, hasParent := s.ParentId(); hasParent {
		req.ParentId = ref(parentID)
	}
	return req
}

// SetHeader will take a TraceInfo and fill out an http.Header, or anything that
// matches the HeaderSetter interface.
func (r TraceInfo) SetHeader(header HeaderSetter) {
	sampled := byte(0)
	if r.Sampled {
		sampled = traceSampled
	}
	if r.TraceId != nil && r.ParentId != nil {
		header.Set(traceParentHeader, fmt.Sprintf("00-%016x-%08x-%x", *r.TraceId, *r.ParentId, int(sampled)))
	} else if r.Sampled {
		header.Set(traceStateHeader, orphanSampling)
	}

}

// hexToUint64 reads a signed int64 that has been formatted as a hex uint64
func hexToUint64(s string) (int64, error) {
	v, err := strconv.ParseUint(s, 16, 64)
	return int64(v), err
}
<commit_msg>http: helper methods to propagate monkit traces (typo fix)<commit_after>// Copyright (C) 2014 Space Monkey, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/spacemonkeygo/monkit/v3"
)

// see: https://github.com/w3c/trace-context/blob/main/spec/20-http_request_header_format.md
const (
	traceSampled      = byte(1)
	traceParentHeader = "traceparent"
	traceStateHeader  = "tracestate"

	// orphanSampling is a special k,v which can be added to the vendor specific tracestate header.
	// it can turn on trace sampling on remote even without propagating the parent trace
	// (traceparent must not contain zero IDs)
	// useful when you use curl (no client side tracing), and would like to get traces from the server.
	orphanSampling = "sampled=true"
)

// TraceInfo is a structure representing an incoming RPC request. Every field
// is optional.
type TraceInfo struct {
	TraceId  *int64
	ParentId *int64
	Sampled  bool
}

// HeaderGetter is an interface that http.Header matches for RequestFromHeader
type HeaderGetter interface {
	Get(string) string
}

// HeaderSetter is an interface that http.Header matches for TraceInfo.SetHeader
type HeaderSetter interface {
	Set(string, string)
}

// TraceInfoFromHeader will create a TraceInfo object given an http.Header or
// anything that matches the HeaderGetter interface.
func TraceInfoFromHeader(header HeaderGetter) (rv TraceInfo) {
	traceParent := header.Get(traceParentHeader)
	traceState := header.Get(traceStateHeader)

	if traceParent != "" {
		parts := strings.Split(traceParent, "-")
		if len(parts) != 4 {
			return rv
		}
		version, err := hexToUint64(parts[0])
		if err != nil || version != 0 {
			return rv
		}
		traceID, err := hexToUint64(parts[1])
		if err != nil {
			return rv
		}
		parentID, err := hexToUint64(parts[2])
		if err != nil {
			return rv
		}
		flags, err := hexToUint64(parts[3])
		if err != nil {
			return rv
		}

		return TraceInfo{
			TraceId:  &traceID,
			ParentId: &parentID,
			Sampled:  (byte(flags) & traceSampled) == traceSampled,
		}
	}

	// trace parent is not set, but tracing can be turned on by a traceState
	if strings.Contains(traceState, "sampled=true") {
		return TraceInfo{
			Sampled: true,
		}
	}
	return rv
}

func ref(v int64) *int64 {
	return &v
}

func TraceInfoFromSpan(s *monkit.Span, sampledf func(trace *monkit.Trace) bool) TraceInfo {
	trace := s.Trace()

	sampled := sampledf(trace)

	if !sampled {
		return TraceInfo{Sampled: sampled}
	}

	req := TraceInfo{
		TraceId:  ref(trace.Id()),
		ParentId: ref(s.Id()),
		Sampled:  sampled,
	}
	if parentID, hasParent := s.ParentId(); hasParent {
		req.ParentId = ref(parentID)
	}
	return req
}

// SetHeader will take a TraceInfo and fill out an http.Header, or anything that
// matches the HeaderSetter interface.
func (r TraceInfo) SetHeader(header HeaderSetter) {
	sampled := byte(0)
	if r.Sampled {
		sampled = traceSampled
	}
	if r.TraceId != nil && r.ParentId != nil {
		header.Set(traceParentHeader, fmt.Sprintf("00-%016x-%08x-%x", *r.TraceId, *r.ParentId, int(sampled)))
	} else if r.Sampled {
		header.Set(traceStateHeader, orphanSampling)
	}

}

// hexToUint64 reads a signed int64 that has been formatted as a hex uint64
func hexToUint64(s string) (int64, error) {
	v, err := strconv.ParseUint(s, 16, 64)
	return int64(v), err
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"path"
	"flag"
	"http"
	"log"
	"strings"
	"fmt"
	"time"
	"recipe"
)

func NotFoundHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Cache-Control", "max-age=10, must-revalidate")
	w.WriteHeader(http.StatusNotFound)
	page := newPage("Not Found")
	page.NotFound = true
	page.Write(w, req)
	return
}

func setCacheControl(w http.ResponseWriter, req *http.Request) {
	if req.Header["X-Draft"] != nil {
		w.Header().Set("Cache-Control", "max-age=1, must-revalidate")
	} else {
		now := time.UTC()
		d := time.Time{2011, 4, 11, 3, 0, 0, time.Monday, 0, "UTC"}
		ONE_WEEK := int64(604800)
		ONE_DAY := int64(86400)
		ttl := ONE_WEEK - (now.Seconds()-d.Seconds())%ONE_DAY
		w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", ttl))
	}
	w.Header().Set("Vary", "Accept-Encoding")
}

func PageHandler(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/" {
		setCacheControl(w, req)
		page := newPage("")
		page.Write(w, req)
		return
	}
	StaticHandler(w, req)
}

func RecipesHandler(w http.ResponseWriter, req *http.Request) {
	var r = recipe.Recipes[path.Base(req.URL.Path)]
	if r != nil {
		var p = strings.Replace(req.URL.Path, "recipes", "recipe", -1)
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}

	if req.URL.Path != "/recipes/" {
		NotFoundHandler(w, req)
		return
	}
	setCacheControl(w, req)
	page := newPage("Recipes")
	page.Recipes = recipe.Recipe_list
	page.Write(w, req)

}

func RecipeHandler(w http.ResponseWriter, req *http.Request) {
	var r = recipe.Recipes[path.Base(req.URL.Path)]
	if r == nil {
		NotFoundHandler(w, req)
		return
	}
	if req.URL.Path[len(req.URL.Path)-1] != '/' {
		http.Redirect(w, req, req.URL.Path+"/", http.StatusMovedPermanently)
		return
	}
	setCacheControl(w, req)
	page := newPage(r.Name + " Recipe")
	page.Recipe = r
	page.Write(w, req)
}

func CanonicalHostHandler(w http.ResponseWriter, req *http.Request) {
	var canonical = "www.eikeon.com"
	if req.Host != canonical {
		http.Redirect(w, req, "http://"+canonical+req.URL.Path, http.StatusMovedPermanently)
	} else {
		NotFoundHandler(w, req)
	}
	// TODO: set CacheControl
}

func StaticHandler(w http.ResponseWriter, req *http.Request) {
	var filename = path.Join(*StaticRoot, req.URL.Path)
	f, err := os.Open(filename)
	if err != nil {
		log.Print(err)
		NotFoundHandler(w, req)
		return
	}
	err = f.Close()

	if strings.Contains(req.URL.Path, "^") {
		w.Header().Set("Cache-Control", "max-age=3153600")
	} else {
		w.Header().Set("Cache-Control", "max-age=1, must-revalidate")
	}
	http.ServeFile(w, req, filename)
}

var Address *string
var StaticRoot *string

func main() {
	Address = flag.String("address", ":9999", "http service address")
	StaticRoot = flag.String("root", "static", "...")
	flag.Parse()

	http.Handle("eikeon.com/", http.HandlerFunc(CanonicalHostHandler))
	http.Handle("/", http.HandlerFunc(PageHandler))
	http.Handle("/recipes/", http.HandlerFunc(RecipesHandler))
	http.Handle("/recipe/", http.HandlerFunc(RecipeHandler))

	err := http.ListenAndServe(*Address, nil)
	if err != nil {
		log.Print("ListenAndServe:", err)
	}
}
<commit_msg>fixed ttl calculation<commit_after>package main

import (
	"os"
	"path"
	"flag"
	"http"
	"log"
	"strings"
	"fmt"
	"time"
	"recipe"
)

func NotFoundHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Cache-Control", "max-age=10, must-revalidate")
	w.WriteHeader(http.StatusNotFound)
	page := newPage("Not Found")
	page.NotFound = true
	page.Write(w, req)
	return
}

func setCacheControl(w http.ResponseWriter, req *http.Request) {
	if req.Header["X-Draft"] != nil {
		w.Header().Set("Cache-Control", "max-age=1, must-revalidate")
	} else {
		now := time.UTC()
		d := time.Time{2011, 4, 11, 3, 0, 0, time.Monday, 0, "UTC"}
		TTL := int64(86400)
		ttl := TTL - (now.Seconds()-d.Seconds())%TTL  // shift
		w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", ttl))
	}
	w.Header().Set("Vary", "Accept-Encoding")
}

func PageHandler(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/" {
		setCacheControl(w, req)
		page := newPage("")
		page.Write(w, req)
		return
	}
	StaticHandler(w, req)
}

func RecipesHandler(w http.ResponseWriter, req *http.Request) {
	var r = recipe.Recipes[path.Base(req.URL.Path)]
	if r != nil {
		var p = strings.Replace(req.URL.Path, "recipes", "recipe", -1)
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}

	if req.URL.Path != "/recipes/" {
		NotFoundHandler(w, req)
		return
	}
	setCacheControl(w, req)
	page := newPage("Recipes")
	page.Recipes = recipe.Recipe_list
	page.Write(w, req)

}

func RecipeHandler(w http.ResponseWriter, req *http.Request) {
	var r = recipe.Recipes[path.Base(req.URL.Path)]
	if r == nil {
		NotFoundHandler(w, req)
		return
	}
	if req.URL.Path[len(req.URL.Path)-1] != '/' {
		http.Redirect(w, req, req.URL.Path+"/", http.StatusMovedPermanently)
		return
	}
	setCacheControl(w, req)
	page := newPage(r.Name + " Recipe")
	page.Recipe = r
	page.Write(w, req)
}

func CanonicalHostHandler(w http.ResponseWriter, req *http.Request) {
	var canonical = "www.eikeon.com"
	if req.Host != canonical {
		http.Redirect(w, req, "http://"+canonical+req.URL.Path, http.StatusMovedPermanently)
	} else {
		NotFoundHandler(w, req)
	}
	// TODO: set CacheControl
}

func StaticHandler(w http.ResponseWriter, req *http.Request) {
	var filename = path.Join(*StaticRoot, req.URL.Path)
	f, err := os.Open(filename)
	if err != nil {
		log.Print(err)
		NotFoundHandler(w, req)
		return
	}
	err = f.Close()

	if strings.Contains(req.URL.Path, "^") {
		w.Header().Set("Cache-Control", "max-age=3153600")
	} else {
		w.Header().Set("Cache-Control", "max-age=1, must-revalidate")
	}
	http.ServeFile(w, req, filename)
}

var Address *string
var StaticRoot *string

func main() {
	Address = flag.String("address", ":9999", "http service address")
	StaticRoot = flag.String("root", "static", "...")
	flag.Parse()

	http.Handle("eikeon.com/", http.HandlerFunc(CanonicalHostHandler))
	http.Handle("/", http.HandlerFunc(PageHandler))
	http.Handle("/recipes/", http.HandlerFunc(RecipesHandler))
	http.Handle("/recipe/", http.HandlerFunc(RecipeHandler))

	err := http.ListenAndServe(*Address, nil)
	if err != nil {
		log.Print("ListenAndServe:", err)
	}
}
<|endoftext|>


<commit_before>// +build darwin

package darwin

import (
	"bufio"
	"bytes"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

/*
collect network interface I/O

`interface.{interface}.{metric}.delta`: The increased amount of network I/O per minute retrieved from the result of netstat -bni

interface = "en0", "en1" and so on...

metric = "rxPackets", "rxErrors", "rxBytes", "txPackets", "txErrors", "txBytes",  "colls"

netstat -bni sample:
Name  Mtu   Network       Address            Ipkts Ierrs     Ibytes    Opkts Oerrs     Obytes  Coll
lo0   16384 <Link#1>                       4504403     0 3063793000  4504403     0 3063793000     0
lo0   16384 localhost   ::1                4504403     - 3063793000  4504403     - 3063793000     -
*/

// InterfaceGenerator XXX
type InterfaceGenerator struct {
	Interval time.Duration
}

// metrics for posting to Mackerel

var interfaceLogger = logging.GetLogger("metrics.interface")

// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectIntarfacesValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectIntarfacesValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = (currValue - value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *InterfaceGenerator) collectIntarfacesValues() (metrics.Values, error) {
	out, err := exec.Command("netstat", "-bni").Output()
	if err != nil {
		interfaceLogger.Errorf("Failed to invoke netstat: %s", err)
		return nil, err
	}

	lineScanner := bufio.NewScanner(bytes.NewReader(out))

	results := make(map[string]float64)
	hasNonZeroValue := false
	for lineScanner.Scan() {
		line := lineScanner.Text()
		fields := strings.Fields(line)
		name := util.SanitizeMetricKey(regexp.MustCompile(`\*`).ReplaceAllString(fields[0], ""))
		if match, _ := regexp.MatchString(`^lo\d+$`, name); match {
			continue
		}
		if match, _ := regexp.MatchString(`^<Link#\d+>$`, fields[2]); match {
			rxIndex, txIndex := getFieldIndex(fields)
			rxKey := fmt.Sprintf("interface.%s.rxBytes", name)
			rxValue, rxErr := strconv.ParseFloat(fields[rxIndex], 64)
			if rxErr != nil {
				interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
				break
			}
			results[rxKey] = rxValue
			if rxValue != 0 {
				hasNonZeroValue = true
			}
			txKey := fmt.Sprintf("interface.%s.txBytes", name)
			txValue, txErr := strconv.ParseFloat(fields[txIndex], 64)
			if txErr != nil {
				interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
				break
			}
			results[txKey] = txValue
			if txValue != 0 {
				hasNonZeroValue = true
			}
		}
	}
	// results (eth0) sample
	/** [%!s(*metrics.Value=&{interface.eth0.rxBytes 6.6074069281e+10})
	     %!s(*metrics.Value=&{interface.eth0.txBytes 9.180531994e+09})
	    ]
	**/
	if hasNonZeroValue {
		return metrics.Values(results), nil
	}
	return nil, nil
}

func getFieldIndex(fields []string) (rxIndex, txIndex int) {
	if len(fields) == 11 {
		rxIndex = 6
		txIndex = 9
	} else {
		rxIndex = 5
		txIndex = 8
	}
	return
}
<commit_msg>more missing import...<commit_after>// +build darwin

package darwin

import (
	"bufio"
	"bytes"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
	"github.com/mackerelio/mackerel-agent/util"
)

/*
collect network interface I/O

`interface.{interface}.{metric}.delta`: The increased amount of network I/O per minute retrieved from the result of netstat -bni

interface = "en0", "en1" and so on...

metric = "rxPackets", "rxErrors", "rxBytes", "txPackets", "txErrors", "txBytes",  "colls"

netstat -bni sample:
Name  Mtu   Network       Address            Ipkts Ierrs     Ibytes    Opkts Oerrs     Obytes  Coll
lo0   16384 <Link#1>                       4504403     0 3063793000  4504403     0 3063793000     0
lo0   16384 localhost   ::1                4504403     - 3063793000  4504403     - 3063793000     -
*/

// InterfaceGenerator XXX
type InterfaceGenerator struct {
	Interval time.Duration
}

// metrics for posting to Mackerel

var interfaceLogger = logging.GetLogger("metrics.interface")

// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectIntarfacesValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectIntarfacesValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = (currValue - value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *InterfaceGenerator) collectIntarfacesValues() (metrics.Values, error) {
	out, err := exec.Command("netstat", "-bni").Output()
	if err != nil {
		interfaceLogger.Errorf("Failed to invoke netstat: %s", err)
		return nil, err
	}

	lineScanner := bufio.NewScanner(bytes.NewReader(out))

	results := make(map[string]float64)
	hasNonZeroValue := false
	for lineScanner.Scan() {
		line := lineScanner.Text()
		fields := strings.Fields(line)
		name := util.SanitizeMetricKey(regexp.MustCompile(`\*`).ReplaceAllString(fields[0], ""))
		if match, _ := regexp.MatchString(`^lo\d+$`, name); match {
			continue
		}
		if match, _ := regexp.MatchString(`^<Link#\d+>$`, fields[2]); match {
			rxIndex, txIndex := getFieldIndex(fields)
			rxKey := fmt.Sprintf("interface.%s.rxBytes", name)
			rxValue, rxErr := strconv.ParseFloat(fields[rxIndex], 64)
			if rxErr != nil {
				interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
				break
			}
			results[rxKey] = rxValue
			if rxValue != 0 {
				hasNonZeroValue = true
			}
			txKey := fmt.Sprintf("interface.%s.txBytes", name)
			txValue, txErr := strconv.ParseFloat(fields[txIndex], 64)
			if txErr != nil {
				interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
				break
			}
			results[txKey] = txValue
			if txValue != 0 {
				hasNonZeroValue = true
			}
		}
	}
	// results (eth0) sample
	/** [%!s(*metrics.Value=&{interface.eth0.rxBytes 6.6074069281e+10})
	     %!s(*metrics.Value=&{interface.eth0.txBytes 9.180531994e+09})
	    ]
	**/
	if hasNonZeroValue {
		return metrics.Values(results), nil
	}
	return nil, nil
}

func getFieldIndex(fields []string) (rxIndex, txIndex int) {
	if len(fields) == 11 {
		rxIndex = 6
		txIndex = 9
	} else {
		rxIndex = 5
		txIndex = 8
	}
	return
}
<|endoftext|>