<commit_before>package paginate

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	DESC = 0
	ASC  = 1
)

type Item interface {
	PaginationValue(p *Pagination) string
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

type Pagination struct {
	Cursor
	defaults Cursor
}

func NewCursorFromQuery(query string) (Cursor, []error) {
	c := Cursor{}
	errors := []error{}
	m, err := url.ParseQuery(query)
	if err != nil {
		errors = append(errors, fmt.Errorf("error when parsing url query string %s: %s", query, err))
		return c, errors
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			errors = append(errors, fmt.Errorf("error when parsing offset %s: %s", v[0], err))
		} else {
			c.Offset = offset
		}
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			errors = append(errors, fmt.Errorf("error when parsing count %s: %s", v[0], err))
		} else {
			c.Count = count
		}
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		direction, err := strconv.Atoi(v[0])
		if err != nil {
			errors = append(errors, fmt.Errorf("error when parsing direction %s: %s", v[0], err))
		} else {
			if direction == ASC || direction == DESC {
				c.Direction = direction
			} else {
				errors = append(errors, fmt.Errorf("'%s' in not a supported direction, use 0 (DESC) or 1 (ASC)", direction))
			}
		}
	}
	return c, errors
}

func (p *Pagination) lastItemIndex(items []Item) int {
	if len(items) <= p.Count {
		return len(items) - 1
	} else {
		// This is the index of the first item on the next page.
		return p.Count
	}
}

func (p *Pagination) equalCount(items []Item, lastItemIndex int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items[i].PaginationValue(p) == items[lastItemIndex].PaginationValue(p) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor, defaults Cursor) *Pagination {
	if defaults.Count == 0 {
		defaults.Count = 10
	}
	if cursor.Count == 0 {
		cursor.Count = defaults.Count
	}

	if cursor.Value == "" {
		cursor.Value = defaults.Value
	}
	if cursor.Offset == 0 {
		cursor.Offset = defaults.Offset
	}
	if cursor.Order == "" {
		cursor.Order = defaults.Order
	}
	if cursor.Direction == 0 {
		cursor.Direction = defaults.Direction
	}

	return &Pagination{cursor, defaults}
}

func NewPaginationFromUrl(rawurl *url.URL, defaults Cursor) (*Pagination, []error) {
	cursor, errors := NewCursorFromQuery(rawurl.RawQuery)
	return NewPagination(cursor, defaults), errors
}

func NewPaginationFromDefaults(defaults Cursor) *Pagination {
	return NewPagination(Cursor{}, defaults)
}

func (p *Pagination) after(items []Item, last, direction int) *Pagination {
	if len(items) == 0 {
		return nil
	}
	value := items[last].PaginationValue(p)
	offset := p.equalCount(items, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return NewPagination(cursor, p.defaults)
}

func (p *Pagination) Prev(items []Item) *Pagination {
	lastItemIndex := 0
	var newDirection int
	if p.Direction == ASC {
		newDirection = DESC
	} else {
		newDirection = ASC
	}
	return p.after(items, lastItemIndex, newDirection)
}

func (p *Pagination) Next(items []Item, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && len(items) <= p.Count {
		return nil
	}
	return p.after(items, p.lastItemIndex(items), p.Direction)
}

func (p *Pagination) ToUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", strconv.Itoa(p.Direction))
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}
<commit_msg>Support lists of any type A, if A implements Item<commit_after>package paginate

import (
	"fmt"
	"net/url"
	"reflect"
	"strconv"
)

const (
	DESC = 0
	ASC  = 1
)

type Item interface {
	PaginationValue(p *Pagination) string
}

func ConvertToItems(its interface{}) ([]Item, error) {
	itsValue := reflect.ValueOf(its)
	itsKind := itsValue.Kind()
	if itsKind != reflect.Array && itsKind != reflect.Slice {
		return nil, fmt.Errorf("Expected items to be an Array or a Slice, got %s", itsKind)
	}
	itsLength := itsValue.Len()
	items := make([]Item, itsLength)
	for i := 0; i < itsLength; i++ {
		itsItem := itsValue.Index(i)
		if item, ok := itsItem.Interface().(Item); ok {
			items[i] = item
		} else {
			return nil, fmt.Errorf("item #%d does not implement the Item interface: %s", i, itsItem)
		}
	}
	return items, nil
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

type Pagination struct {
	Cursor
	defaults Cursor
}

func NewCursorFromQuery(query string) (Cursor, []error) {
	c := Cursor{}
	errors := []error{}
	m, err := url.ParseQuery(query)
	if err != nil {
		errors = append(errors, fmt.Errorf("error when parsing url query string %s: %s", query, err))
		return c, errors
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			errors = append(errors, fmt.Errorf("error when parsing offset %s: %s", v[0], err))
		} else {
			c.Offset = offset
		}
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			errors = append(errors, fmt.Errorf("error when parsing count %s: %s", v[0], err))
		} else {
			c.Count = count
		}
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		direction, err := strconv.Atoi(v[0])
		if err != nil {
			errors = append(errors, fmt.Errorf("error when parsing direction %s: %s", v[0], err))
		} else {
			if direction == ASC || direction == DESC {
				c.Direction = direction
			} else {
				errors = append(errors, fmt.Errorf("'%s' in not a supported direction, use 0 (DESC) or 1 (ASC)", direction))
			}
		}
	}
	return c, errors
}

func (p *Pagination) lastItemIndex(items []Item) int {
	if len(items) <= p.Count {
		return len(items) - 1
	} else {
		// This is the index of the first item on the next page.
		return p.Count
	}
}

func (p *Pagination) equalCount(items []Item, lastItemIndex int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items[i].PaginationValue(p) == items[lastItemIndex].PaginationValue(p) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor, defaults Cursor) *Pagination {
	if defaults.Count == 0 {
		defaults.Count = 10
	}
	if cursor.Count == 0 {
		cursor.Count = defaults.Count
	}

	if cursor.Value == "" {
		cursor.Value = defaults.Value
	}
	if cursor.Offset == 0 {
		cursor.Offset = defaults.Offset
	}
	if cursor.Order == "" {
		cursor.Order = defaults.Order
	}
	if cursor.Direction == 0 {
		cursor.Direction = defaults.Direction
	}

	return &Pagination{cursor, defaults}
}

func NewPaginationFromUrl(rawurl *url.URL, defaults Cursor) (*Pagination, []error) {
	cursor, errors := NewCursorFromQuery(rawurl.RawQuery)
	return NewPagination(cursor, defaults), errors
}

func NewPaginationFromDefaults(defaults Cursor) *Pagination {
	return NewPagination(Cursor{}, defaults)
}

func (p *Pagination) after(items []Item, last, direction int) *Pagination {
	if len(items) == 0 {
		return nil
	}
	value := items[last].PaginationValue(p)
	offset := p.equalCount(items, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return NewPagination(cursor, p.defaults)
}

func (p *Pagination) Prev(its interface{}) (*Pagination, error) {
	items, err := ConvertToItems(its)
	if err != nil {
		return nil, err
	}
	lastItemIndex := 0
	var newDirection int
	if p.Direction == ASC {
		newDirection = DESC
	} else {
		newDirection = ASC
	}
	return p.after(items, lastItemIndex, newDirection), nil
}

func (p *Pagination) Next(its interface{}, next_page_prefetched bool) (*Pagination, error) {
	items, err := ConvertToItems(its)
	if err != nil {
		return nil, err
	}
	if next_page_prefetched && len(items) <= p.Count {
		return nil, nil
	}
	return p.after(items, p.lastItemIndex(items), p.Direction), nil
}

func (p *Pagination) ToUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", strconv.Itoa(p.Direction))
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}
<|endoftext|>


<commit_before>package async

/*

Parallel is shorthand function to List.RunParallel without having to manually
create a new list, add the routines, etc.

*/
func Parallel(routines []Routine, callbacks ...Done) {
  l := New()
  l.Multiple(routines...)

  l.RunParallel(callbacks...)
}

/*

RunParallel will run all of the Routine functions from the current list in
parallel mode.

All of the arguments returned in a Routine's Done function will be combined
and returned in the callbacks that are provided.

If there is an error, any further results will be discarded but it will not
immediately exit. It will continue to run all of the other Routine functions
that were passed into it. This is because by the time the error is sent, the
goroutines have already been started. At this current time, there is no way
to cancel a sleep timer in Go.

For example:
  async.Parallel([]async.Routine{
    func(done async.Done, args ...interface{}) {
      time.Sleep(20 * time.Second)
      done(nil, "Won't trigger the callbacks because error has been sent")
    },
    func(done async.Done, args ...interface{}) {
      done(fmt.Errorf("Test error"))
    }
  }, func(err error, results ...interface{}) {
    if err != nil {
      fmt.Printf("Error: %s", err)
      return
    }

    fmt.Printf("Args: %s", args)
  })

If you were to run this example, you would see the error happen immediately.
However, you would also notice that the program doesn't immediately exit.
That is because it is still waiting for responses that it silently discards,
since an error has already occurred.

*/
func (l *List) RunParallel(callbacks ...Done) {
  var (
    results = make([]interface{}, 0)

    result = make(chan interface{})

    _error error

    final = func(err error, results ...interface{}) {
      for i := 0; i < len(callbacks); i++ {
        if err != nil {
          callbacks[i](err)
        } else {
          callbacks[i](err, results...)
        }
      }
    }
  )

  defer close(result)

  l.Wait.Add(l.Len())

  go func() {
    for {
      r := <-result

      if _error != nil {
        continue
      }

      switch r.(type) {
      case error:
        _error = r.(error)
        final(_error)

      case []interface{}:
        results = append(results, r.([]interface{})...)
      }
    }
  }()

  for l.Len() > 0 {
    e := l.Front()
    _, r := l.Remove(e)

    go r(func(err error, args ...interface{}) {
      defer l.Wait.Done()

      if err != nil {
        result <- err
        return
      }

      result <- args
    })
  }

  l.Wait.Wait()

  if _error == nil {
    final(nil, results...)
  }
}
<commit_msg>Fix typo in documentation<commit_after>package async

/*

Parallel is a shorthand function to List.RunParallel without having to
manually create a new list, add the routines, etc.

*/
func Parallel(routines []Routine, callbacks ...Done) {
  l := New()
  l.Multiple(routines...)

  l.RunParallel(callbacks...)
}

/*

RunParallel will run all of the Routine functions from the current list in
parallel mode.

All of the arguments returned in a Routine's Done function will be combined
and returned in the callbacks that are provided.

If there is an error, any further results will be discarded but it will not
immediately exit. It will continue to run all of the other Routine functions
that were passed into it. This is because by the time the error is sent, the
goroutines have already been started. At this current time, there is no way
to cancel a sleep timer in Go.

For example:
  async.Parallel([]async.Routine{
    func(done async.Done, args ...interface{}) {
      time.Sleep(20 * time.Second)
      done(nil, "Won't trigger the callbacks because error has been sent")
    },
    func(done async.Done, args ...interface{}) {
      done(fmt.Errorf("Test error"))
    }
  }, func(err error, results ...interface{}) {
    if err != nil {
      fmt.Printf("Error: %s", err)
      return
    }

    fmt.Printf("Args: %s", args)
  })

If you were to run this example, you would see the error happen immediately.
However, you would also notice that the program doesn't immediately exit.
That is because it is still waiting for responses that it silently discards,
since an error has already occurred.

*/
func (l *List) RunParallel(callbacks ...Done) {
  var (
    results = make([]interface{}, 0)

    result = make(chan interface{})

    _error error

    final = func(err error, results ...interface{}) {
      for i := 0; i < len(callbacks); i++ {
        if err != nil {
          callbacks[i](err)
        } else {
          callbacks[i](err, results...)
        }
      }
    }
  )

  defer close(result)

  l.Wait.Add(l.Len())

  go func() {
    for {
      r := <-result

      if _error != nil {
        continue
      }

      switch r.(type) {
      case error:
        _error = r.(error)
        final(_error)

      case []interface{}:
        results = append(results, r.([]interface{})...)
      }
    }
  }()

  for l.Len() > 0 {
    e := l.Front()
    _, r := l.Remove(e)

    go r(func(err error, args ...interface{}) {
      defer l.Wait.Done()

      if err != nil {
        result <- err
        return
      }

      result <- args
    })
  }

  l.Wait.Wait()

  if _error == nil {
    final(nil, results...)
  }
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright (c) 2020  All rights reserved.

package tc

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/libcalico-go/lib/set"

	"github.com/projectcalico/felix/bpf"
)

type AttachPoint struct {
	Type       EndpointType
	ToOrFrom   ToOrFromEp
	Hook       Hook
	Iface      string
	LogLevel   string
	HostIP     net.IP
	FIB        bool
	ToHostDrop bool
	DSR        bool
	TunnelMTU  uint16
}

var tcLock sync.RWMutex

var ErrDeviceNotFound = errors.New("device not found")
var prefHandleRe = regexp.MustCompile(`pref ([^ ]+) .* handle ([^ ]+)`)

// AttachProgram attaches a BPF program from a file to the TC attach point
func (ap AttachPoint) AttachProgram() error {
	logCxt := log.WithField("attachPoint", ap)

	tempDir, err := ioutil.TempDir("", "calico-tc")
	if err != nil {
		return fmt.Errorf("failed to create temporary directory: %w", err)
	}
	defer func() {
		_ = os.RemoveAll(tempDir)
	}()

	filename := ap.FileName()
	preCompiledBinary := path.Join(bpf.ObjectDir, filename)
	tempBinary := path.Join(tempDir, filename)

	err = ap.patchBinary(logCxt, preCompiledBinary, tempBinary)
	if err != nil {
		logCxt.WithError(err).Error("Failed to patch binary")
		return err
	}

	// Using the RLock allows multiple attach calls to proceed in parallel unless
	// CleanUpJumpMaps() (which takes the writer lock) is running.
	logCxt.Debug("AttachProgram waiting for lock...")
	tcLock.RLock()
	defer tcLock.RUnlock()
	logCxt.Debug("AttachProgram got lock.")

	progsToClean, err := ap.listAttachedPrograms()
	if err != nil {
		return err
	}

	_, err = tc("filter", "add", "dev", ap.Iface, string(ap.Hook),
		"bpf", "da", "obj", tempBinary,
		"sec", SectionName(ap.Type, ap.ToOrFrom),
	)
	if err != nil {
		return err
	}

	// Success: clean up the old programs.
	var progErrs []error
	for _, p := range progsToClean {
		log.WithField("prog", p).Debug("Cleaning up old calico program")
		_, err = tc("filter", "del", "dev", ap.Iface, string(ap.Hook), "pref", p.pref, "handle", p.handle, "bpf")
		if err == ErrDeviceNotFound {
			continue
		}
		if err != nil {
			log.WithError(err).WithField("prog", p).Warn("Failed to clean up old calico program.")
			progErrs = append(progErrs, err)
		}
	}

	if len(progErrs) != 0 {
		return fmt.Errorf("failed to clean up one or more old calico programs: %v", progErrs)
	}

	return nil
}

func tc(args ...string) (out string, err error) {
	tcCmd := exec.Command("tc", args...)
	outBytes, err := tcCmd.Output()
	if err != nil {
		if isCannotFindDevice(err) {
			err = ErrDeviceNotFound
		} else if err2, ok := err.(*exec.ExitError); ok {
			err = fmt.Errorf("failed to execute tc %v: rc=%v stderr=%v (%w)",
				args, err2.ExitCode(), string(err2.Stderr), err)
		} else {
			err = fmt.Errorf("failed to execute tc %v: %w", args, err)
		}
	}
	out = string(outBytes)
	return
}

func isCannotFindDevice(err error) bool {
	if errors.Is(err, ErrDeviceNotFound) {
		return true
	}
	if err, ok := err.(*exec.ExitError); ok {
		stderr := string(err.Stderr)
		if strings.Contains(stderr, "Cannot find device") {
			return true
		}
	}
	return false
}

type attachedProg struct {
	pref   string
	handle string
}

func (ap AttachPoint) listAttachedPrograms() ([]attachedProg, error) {
	out, err := tc("filter", "show", "dev", ap.Iface, string(ap.Hook))
	if err != nil {
		return nil, fmt.Errorf("failed to list tc filters on interface: %w", err)
	}
	// Lines look like this; the section name always includes calico.
	// filter protocol all pref 49152 bpf chain 0 handle 0x1 to_hep_no_log.o:[calico_to_host_ep] direct-action not_in_hw id 821 tag ee402594f8f85ac3 jited
	var progsToClean []attachedProg
	for _, line := range strings.Split(string(out), "\n") {
		if !strings.Contains(line, "calico") {
			continue
		}
		// find the pref and the handle
		if sm := prefHandleRe.FindStringSubmatch(line); len(sm) > 0 {
			p := attachedProg{
				pref:   sm[1],
				handle: sm[2],
			}
			log.WithField("prog", p).Debug("Found old calico program")
			progsToClean = append(progsToClean, p)
		}
	}
	return progsToClean, nil
}

func (ap AttachPoint) patchBinary(logCtx *log.Entry, ifile, ofile string) error {
	b, err := bpf.BinaryFromFile(ifile)
	if err != nil {
		return fmt.Errorf("failed to read pre-compiled BPF binary: %w", err)
	}

	logCtx.WithField("ip", ap.HostIP).Debug("Patching in IP")
	err = b.PatchIPv4(ap.HostIP)
	if err != nil {
		return fmt.Errorf("failed to patch IPv4 into BPF binary: %w", err)
	}

	b.PatchLogPrefix(ap.Iface)
	b.PatchTunnelMTU(ap.TunnelMTU)

	err = b.WriteToFile(ofile)
	if err != nil {
		return fmt.Errorf("failed to write pre-compiled BPF binary: %w", err)
	}

	return nil
}

// ProgramName returnt the name of the program associated with this AttachPoint
func (ap AttachPoint) ProgramName() string {
	return SectionName(ap.Type, ap.ToOrFrom)
}

// FileName return the file the AttachPoint will load the program from
func (ap AttachPoint) FileName() string {
	return ProgFilename(ap.Type, ap.ToOrFrom, ap.ToHostDrop, ap.FIB, ap.DSR, ap.LogLevel)
}

func (ap AttachPoint) IsAttached() (bool, error) {
	hasQ, err := HasQdisc(ap.Iface)
	if err != nil {
		return false, err
	}
	if !hasQ {
		return false, nil
	}
	progs, err := ap.listAttachedPrograms()
	if err != nil {
		return false, err
	}
	return len(progs) > 0, nil
}

// tcDirRegex matches tc's auto-created directory names so we can clean them up when removing maps without accidentally
// removing other user-created dirs..
var tcDirRegex = regexp.MustCompile(`[0-9a-f]{40}`)

// CleanUpJumpMaps scans for cali_jump maps that are still pinned to the filesystem but no longer referenced by
// our BPF programs.
func CleanUpJumpMaps() {
	// So that we serialise with AttachProgram()
	log.Debug("CleanUpJumpMaps waiting for lock...")
	tcLock.Lock()
	defer tcLock.Unlock()
	log.Debug("CleanUpJumpMaps got lock, cleaning up...")

	// Find the maps we care about by walking the BPF filesystem.
	mapIDToPath := make(map[int]string)
	err := filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if strings.HasPrefix(info.Name(), "cali_jump") {
			log.WithField("path", p).Debug("Examining map")

			out, err := exec.Command("bpftool", "map", "show", "pinned", p).Output()
			if err != nil {
				log.WithError(err).Panic("Failed to show map")
			}
			log.WithField("dump", string(out)).Debug("Map show before deletion")
			idStr := string(bytes.Split(out, []byte(":"))[0])
			id, err := strconv.Atoi(idStr)
			if err != nil {
				log.WithError(err).WithField("dump", string(out)).Error("Failed to parse bpftool output.")
				return err
			}
			mapIDToPath[id] = p
		}
		return nil
	})
	if os.IsNotExist(err) {
		log.WithError(err).Warn("tc directory missing from BPF file system?")
		return
	}
	if err != nil {
		log.WithError(err).Error("Error while looking for maps.")
	}

	// Find all the programs that are attached to interfaces.
	out, err := exec.Command("bpftool", "net", "-j").Output()
	if err != nil {
		log.WithError(err).Panic("Failed to list attached bpf programs")
	}
	log.WithField("dump", string(out)).Debug("Attached BPF programs")

	var attached []struct {
		TC []struct {
			DevName string `json:"devname"`
			ID      int    `json:"id"`
		} `json:"tc"`
	}
	err = json.Unmarshal(out, &attached)
	if err != nil {
		log.WithError(err).WithField("dump", string(out)).Error("Failed to parse list of attached BPF programs")
	}
	attachedProgs := set.New()
	for _, prog := range attached[0].TC {
		log.WithField("prog", prog).Debug("Adding prog to attached set")
		attachedProgs.Add(prog.ID)
	}

	// Find all the maps that the attached programs refer to and remove them from consideration.
	progsJSON, err := exec.Command("bpftool", "prog", "list", "--json").Output()
	if err != nil {
		log.WithError(err).Info("Failed to list BPF programs, assuming there's nothing to clean up.")
		return
	}
	var progs []struct {
		ID   int    `json:"id"`
		Name string `json:"name"`
		Maps []int  `json:"map_ids"`
	}
	err = json.Unmarshal(progsJSON, &progs)
	if err != nil {
		log.WithError(err).Info("Failed to parse bpftool output.  Assuming nothing to clean up.")
		return
	}
	for _, p := range progs {
		if !attachedProgs.Contains(p.ID) {
			log.WithField("prog", p).Debug("Prog is not in the attached set, skipping")
			continue
		}
		for _, id := range p.Maps {
			log.WithField("mapID", id).WithField("prog", p).Debug("Map is still in use")
			delete(mapIDToPath, id)
		}
	}

	// Remove the pins.
	for id, p := range mapIDToPath {
		log.WithFields(log.Fields{"id": id, "path": p}).Debug("Removing stale BPF map pin.")
		err := os.Remove(p)
		if err != nil {
			log.WithError(err).Warn("Removed stale BPF map pin.")
		}
		log.WithFields(log.Fields{"id": id, "path": p}).Info("Removed stale BPF map pin.")
	}

	// Look for empty dirs.
	emptyAutoDirs := set.New()
	err = filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && tcDirRegex.MatchString(info.Name()) {
			p := path.Clean(p)
			log.WithField("path", p).Debug("Found tc auto-created dir.")
			emptyAutoDirs.Add(p)
		} else {
			dirPath := path.Clean(path.Dir(p))
			if emptyAutoDirs.Contains(dirPath) {
				log.WithField("path", dirPath).Debug("tc dir is not empty.")
				emptyAutoDirs.Discard(dirPath)
			}
		}
		return nil
	})
	if os.IsNotExist(err) {
		log.WithError(err).Warn("tc directory missing from BPF file system?")
		return
	}
	if err != nil {
		log.WithError(err).Error("Error while looking for maps.")
	}

	emptyAutoDirs.Iter(func(item interface{}) error {
		p := item.(string)
		log.WithField("path", p).Debug("Removing empty dir.")
		err := os.Remove(p)
		if err != nil {
			log.WithError(err).Error("Error while removing empty dir.")
		}
		return nil
	})
}

// EnsureQdisc makes sure that qdisc is attached to the given interface
func EnsureQdisc(ifaceName string) error {
	hasQdisc, err := HasQdisc(ifaceName)
	if err != nil {
		return err
	}
	if hasQdisc {
		log.WithField("iface", ifaceName).Debug("Already have a clsact qdisc on this interface")
		return nil
	}
	_, err = tc("qdisc", "add", "dev", ifaceName, "clsact")
	if err != nil {
		return fmt.Errorf("failed to add qdisc to interface '%s': %w", ifaceName, err)
	}
	return nil
}

func HasQdisc(ifaceName string) (bool, error) {
	out, err := tc("qdisc", "show", "dev", ifaceName, "clsact")
	if err != nil {
		return false, fmt.Errorf("failed to check if interface '%s' has qdisc: %w", ifaceName, err)
	}
	if strings.Contains(out, "qdisc clsact") {
		return true, nil
	}
	return false, nil
}

// RemoveQdisc makes sure that there is no qdisc attached to the given interface
func RemoveQdisc(ifaceName string) error {
	hasQdisc, err := HasQdisc(ifaceName)
	if err != nil {
		return err
	}
	if !hasQdisc {
		return nil
	}
	_, err = tc("qdisc", "del", "dev", ifaceName, "clsact")
	if err != nil {
		return fmt.Errorf("failed to remove qdisc from interface '%s': %w", ifaceName, err)
	}
	return nil
}
<commit_msg>Detect another possible tc error output.<commit_after>// Copyright (c) 2020 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright (c) 2020  All rights reserved.

package tc

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/libcalico-go/lib/set"

	"github.com/projectcalico/felix/bpf"
)

type AttachPoint struct {
	Type       EndpointType
	ToOrFrom   ToOrFromEp
	Hook       Hook
	Iface      string
	LogLevel   string
	HostIP     net.IP
	FIB        bool
	ToHostDrop bool
	DSR        bool
	TunnelMTU  uint16
}

var tcLock sync.RWMutex

var ErrDeviceNotFound = errors.New("device not found")
var prefHandleRe = regexp.MustCompile(`pref ([^ ]+) .* handle ([^ ]+)`)

// AttachProgram attaches a BPF program from a file to the TC attach point
func (ap AttachPoint) AttachProgram() error {
	logCxt := log.WithField("attachPoint", ap)

	tempDir, err := ioutil.TempDir("", "calico-tc")
	if err != nil {
		return fmt.Errorf("failed to create temporary directory: %w", err)
	}
	defer func() {
		_ = os.RemoveAll(tempDir)
	}()

	filename := ap.FileName()
	preCompiledBinary := path.Join(bpf.ObjectDir, filename)
	tempBinary := path.Join(tempDir, filename)

	err = ap.patchBinary(logCxt, preCompiledBinary, tempBinary)
	if err != nil {
		logCxt.WithError(err).Error("Failed to patch binary")
		return err
	}

	// Using the RLock allows multiple attach calls to proceed in parallel unless
	// CleanUpJumpMaps() (which takes the writer lock) is running.
	logCxt.Debug("AttachProgram waiting for lock...")
	tcLock.RLock()
	defer tcLock.RUnlock()
	logCxt.Debug("AttachProgram got lock.")

	progsToClean, err := ap.listAttachedPrograms()
	if err != nil {
		return err
	}

	_, err = tc("filter", "add", "dev", ap.Iface, string(ap.Hook),
		"bpf", "da", "obj", tempBinary,
		"sec", SectionName(ap.Type, ap.ToOrFrom),
	)
	if err != nil {
		return err
	}

	// Success: clean up the old programs.
	var progErrs []error
	for _, p := range progsToClean {
		log.WithField("prog", p).Debug("Cleaning up old calico program")
		_, err = tc("filter", "del", "dev", ap.Iface, string(ap.Hook), "pref", p.pref, "handle", p.handle, "bpf")
		if err == ErrDeviceNotFound {
			continue
		}
		if err != nil {
			log.WithError(err).WithField("prog", p).Warn("Failed to clean up old calico program.")
			progErrs = append(progErrs, err)
		}
	}

	if len(progErrs) != 0 {
		return fmt.Errorf("failed to clean up one or more old calico programs: %v", progErrs)
	}

	return nil
}

func tc(args ...string) (out string, err error) {
	tcCmd := exec.Command("tc", args...)
	outBytes, err := tcCmd.Output()
	if err != nil {
		if isCannotFindDevice(err) {
			err = ErrDeviceNotFound
		} else if err2, ok := err.(*exec.ExitError); ok {
			err = fmt.Errorf("failed to execute tc %v: rc=%v stderr=%v (%w)",
				args, err2.ExitCode(), string(err2.Stderr), err)
		} else {
			err = fmt.Errorf("failed to execute tc %v: %w", args, err)
		}
	}
	out = string(outBytes)
	return
}

func isCannotFindDevice(err error) bool {
	if errors.Is(err, ErrDeviceNotFound) {
		return true
	}
	if err, ok := err.(*exec.ExitError); ok {
		stderr := string(err.Stderr)
		if strings.Contains(stderr, "Cannot find device") ||
			strings.Contains(stderr, "No such device") {
			return true
		}
	}
	return false
}

type attachedProg struct {
	pref   string
	handle string
}

func (ap AttachPoint) listAttachedPrograms() ([]attachedProg, error) {
	out, err := tc("filter", "show", "dev", ap.Iface, string(ap.Hook))
	if err != nil {
		return nil, fmt.Errorf("failed to list tc filters on interface: %w", err)
	}
	// Lines look like this; the section name always includes calico.
	// filter protocol all pref 49152 bpf chain 0 handle 0x1 to_hep_no_log.o:[calico_to_host_ep] direct-action not_in_hw id 821 tag ee402594f8f85ac3 jited
	var progsToClean []attachedProg
	for _, line := range strings.Split(string(out), "\n") {
		if !strings.Contains(line, "calico") {
			continue
		}
		// find the pref and the handle
		if sm := prefHandleRe.FindStringSubmatch(line); len(sm) > 0 {
			p := attachedProg{
				pref:   sm[1],
				handle: sm[2],
			}
			log.WithField("prog", p).Debug("Found old calico program")
			progsToClean = append(progsToClean, p)
		}
	}
	return progsToClean, nil
}

func (ap AttachPoint) patchBinary(logCtx *log.Entry, ifile, ofile string) error {
	b, err := bpf.BinaryFromFile(ifile)
	if err != nil {
		return fmt.Errorf("failed to read pre-compiled BPF binary: %w", err)
	}

	logCtx.WithField("ip", ap.HostIP).Debug("Patching in IP")
	err = b.PatchIPv4(ap.HostIP)
	if err != nil {
		return fmt.Errorf("failed to patch IPv4 into BPF binary: %w", err)
	}

	b.PatchLogPrefix(ap.Iface)
	b.PatchTunnelMTU(ap.TunnelMTU)

	err = b.WriteToFile(ofile)
	if err != nil {
		return fmt.Errorf("failed to write pre-compiled BPF binary: %w", err)
	}

	return nil
}

// ProgramName returnt the name of the program associated with this AttachPoint
func (ap AttachPoint) ProgramName() string {
	return SectionName(ap.Type, ap.ToOrFrom)
}

// FileName return the file the AttachPoint will load the program from
func (ap AttachPoint) FileName() string {
	return ProgFilename(ap.Type, ap.ToOrFrom, ap.ToHostDrop, ap.FIB, ap.DSR, ap.LogLevel)
}

func (ap AttachPoint) IsAttached() (bool, error) {
	hasQ, err := HasQdisc(ap.Iface)
	if err != nil {
		return false, err
	}
	if !hasQ {
		return false, nil
	}
	progs, err := ap.listAttachedPrograms()
	if err != nil {
		return false, err
	}
	return len(progs) > 0, nil
}

// tcDirRegex matches tc's auto-created directory names so we can clean them up when removing maps without accidentally
// removing other user-created dirs..
var tcDirRegex = regexp.MustCompile(`[0-9a-f]{40}`)

// CleanUpJumpMaps scans for cali_jump maps that are still pinned to the filesystem but no longer referenced by
// our BPF programs.
func CleanUpJumpMaps() {
	// So that we serialise with AttachProgram()
	log.Debug("CleanUpJumpMaps waiting for lock...")
	tcLock.Lock()
	defer tcLock.Unlock()
	log.Debug("CleanUpJumpMaps got lock, cleaning up...")

	// Find the maps we care about by walking the BPF filesystem.
	mapIDToPath := make(map[int]string)
	err := filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if strings.HasPrefix(info.Name(), "cali_jump") {
			log.WithField("path", p).Debug("Examining map")

			out, err := exec.Command("bpftool", "map", "show", "pinned", p).Output()
			if err != nil {
				log.WithError(err).Panic("Failed to show map")
			}
			log.WithField("dump", string(out)).Debug("Map show before deletion")
			idStr := string(bytes.Split(out, []byte(":"))[0])
			id, err := strconv.Atoi(idStr)
			if err != nil {
				log.WithError(err).WithField("dump", string(out)).Error("Failed to parse bpftool output.")
				return err
			}
			mapIDToPath[id] = p
		}
		return nil
	})
	if os.IsNotExist(err) {
		log.WithError(err).Warn("tc directory missing from BPF file system?")
		return
	}
	if err != nil {
		log.WithError(err).Error("Error while looking for maps.")
	}

	// Find all the programs that are attached to interfaces.
	out, err := exec.Command("bpftool", "net", "-j").Output()
	if err != nil {
		log.WithError(err).Panic("Failed to list attached bpf programs")
	}
	log.WithField("dump", string(out)).Debug("Attached BPF programs")

	var attached []struct {
		TC []struct {
			DevName string `json:"devname"`
			ID      int    `json:"id"`
		} `json:"tc"`
	}
	err = json.Unmarshal(out, &attached)
	if err != nil {
		log.WithError(err).WithField("dump", string(out)).Error("Failed to parse list of attached BPF programs")
	}
	attachedProgs := set.New()
	for _, prog := range attached[0].TC {
		log.WithField("prog", prog).Debug("Adding prog to attached set")
		attachedProgs.Add(prog.ID)
	}

	// Find all the maps that the attached programs refer to and remove them from consideration.
	progsJSON, err := exec.Command("bpftool", "prog", "list", "--json").Output()
	if err != nil {
		log.WithError(err).Info("Failed to list BPF programs, assuming there's nothing to clean up.")
		return
	}
	var progs []struct {
		ID   int    `json:"id"`
		Name string `json:"name"`
		Maps []int  `json:"map_ids"`
	}
	err = json.Unmarshal(progsJSON, &progs)
	if err != nil {
		log.WithError(err).Info("Failed to parse bpftool output.  Assuming nothing to clean up.")
		return
	}
	for _, p := range progs {
		if !attachedProgs.Contains(p.ID) {
			log.WithField("prog", p).Debug("Prog is not in the attached set, skipping")
			continue
		}
		for _, id := range p.Maps {
			log.WithField("mapID", id).WithField("prog", p).Debug("Map is still in use")
			delete(mapIDToPath, id)
		}
	}

	// Remove the pins.
	for id, p := range mapIDToPath {
		log.WithFields(log.Fields{"id": id, "path": p}).Debug("Removing stale BPF map pin.")
		err := os.Remove(p)
		if err != nil {
			log.WithError(err).Warn("Removed stale BPF map pin.")
		}
		log.WithFields(log.Fields{"id": id, "path": p}).Info("Removed stale BPF map pin.")
	}

	// Look for empty dirs.
	emptyAutoDirs := set.New()
	err = filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && tcDirRegex.MatchString(info.Name()) {
			p := path.Clean(p)
			log.WithField("path", p).Debug("Found tc auto-created dir.")
			emptyAutoDirs.Add(p)
		} else {
			dirPath := path.Clean(path.Dir(p))
			if emptyAutoDirs.Contains(dirPath) {
				log.WithField("path", dirPath).Debug("tc dir is not empty.")
				emptyAutoDirs.Discard(dirPath)
			}
		}
		return nil
	})
	if os.IsNotExist(err) {
		log.WithError(err).Warn("tc directory missing from BPF file system?")
		return
	}
	if err != nil {
		log.WithError(err).Error("Error while looking for maps.")
	}

	emptyAutoDirs.Iter(func(item interface{}) error {
		p := item.(string)
		log.WithField("path", p).Debug("Removing empty dir.")
		err := os.Remove(p)
		if err != nil {
			log.WithError(err).Error("Error while removing empty dir.")
		}
		return nil
	})
}

// EnsureQdisc makes sure that qdisc is attached to the given interface
func EnsureQdisc(ifaceName string) error {
	hasQdisc, err := HasQdisc(ifaceName)
	if err != nil {
		return err
	}
	if hasQdisc {
		log.WithField("iface", ifaceName).Debug("Already have a clsact qdisc on this interface")
		return nil
	}
	_, err = tc("qdisc", "add", "dev", ifaceName, "clsact")
	if err != nil {
		return fmt.Errorf("failed to add qdisc to interface '%s': %w", ifaceName, err)
	}
	return nil
}

func HasQdisc(ifaceName string) (bool, error) {
	out, err := tc("qdisc", "show", "dev", ifaceName, "clsact")
	if err != nil {
		return false, fmt.Errorf("failed to check if interface '%s' has qdisc: %w", ifaceName, err)
	}
	if strings.Contains(out, "qdisc clsact") {
		return true, nil
	}
	return false, nil
}

// RemoveQdisc makes sure that there is no qdisc attached to the given interface
func RemoveQdisc(ifaceName string) error {
	hasQdisc, err := HasQdisc(ifaceName)
	if err != nil {
		return err
	}
	if !hasQdisc {
		return nil
	}
	_, err = tc("qdisc", "del", "dev", ifaceName, "clsact")
	if err != nil {
		return fmt.Errorf("failed to remove qdisc from interface '%s': %w", ifaceName, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package kafka

import "sync"

type topicPartition struct {
	topic     string
	partition int32
}

type brokerManager struct {
	client        *Client
	defaultBroker *broker

	brokers     map[int32]*broker
	leaders     map[topicPartition]int32
	brokersLock sync.RWMutex
}

func newBrokerManager(client *Client, host string, port int32) (bm *brokerManager, err error) {
	bm = new(brokerManager)

	bm.client = client

	// we create a new broker object as the default 'master' broker
	// if this broker is also a leader then we will end up with two broker objects for it, but that's not a big deal
	bm.defaultBroker, err = newBroker(host, port)
	if err != nil {
		return nil, err
	}

	bm.brokers = make(map[int32]*broker)
	bm.leaders = make(map[topicPartition]int32)

	// do an initial fetch of all cluster metadata by specifing an empty list of topics
	err = bm.refreshTopics(make([]*string, 0))
	if err != nil {
		return nil, err
	}

	return bm, nil
}

func (bm *brokerManager) lookupLeader(topic string, partition int32) (*broker, error) {
	var broker *broker = nil
	bm.brokersLock.RLock()
	id, ok := bm.leaders[topicPartition{topic, partition}]
	if ok {
		broker = bm.brokers[id]
	}
	bm.brokersLock.RUnlock()

	if broker == nil {
		err := bm.refreshTopic(topic)
		if err != nil {
			return nil, err
		}
		bm.brokersLock.RLock()
		broker = bm.brokers[bm.leaders[topicPartition{topic, partition}]]
		bm.brokersLock.RUnlock()
	}

	if broker == nil {
		return nil, UNKNOWN_TOPIC_OR_PARTITION
	}

	return broker, nil
}

func (bm *brokerManager) getDefault() *broker {

	if bm.defaultBroker == nil {
		bm.brokersLock.RLock()
		defer bm.brokersLock.RUnlock()
		for _, id := range bm.leaders {
			bm.defaultBroker = bm.brokers[id]
			break
		}
	}

	return bm.defaultBroker
}

func (bm *brokerManager) tryDefaultBrokers(req encoder, res decoder) error {
	for b := bm.getDefault(); b != nil; b = bm.getDefault() {
		responseChan, err := b.sendRequest(bm.client.id, req)
		if err != nil {
			return err
		}

		select {
		case buf := <-responseChan.packets:
			decoder := realDecoder{raw: buf}
			err = res.decode(&decoder)
			return err
		case <-responseChan.errors:
			bm.defaultBroker = nil
			bm.brokersLock.Lock()
			delete(bm.brokers, b.id)
			bm.brokersLock.Unlock()
		}
	}
	return OutOfBrokers{}
}

func (bm *brokerManager) refreshTopics(topics []*string) error {
	response := new(metadata)
	err := bm.tryDefaultBrokers(&metadataRequest{topics}, response)
	if err != nil {
		return err
	}

	bm.brokersLock.Lock()
	defer bm.brokersLock.Unlock()

	for i := range response.brokers {
		broker := &response.brokers[i]
		bm.brokers[broker.id] = broker
	}

	for i := range response.topics {
		topic := &response.topics[i]
		if topic.err != NO_ERROR {
			return topic.err
		}
		for j := range topic.partitions {
			partition := &topic.partitions[j]
			if partition.err != NO_ERROR {
				return partition.err
			}
			bm.leaders[topicPartition{*topic.name, partition.id}] = partition.leader
		}
	}

	return nil
}

func (bm *brokerManager) refreshTopic(topic string) error {
	tmp := make([]*string, 1)
	tmp[0] = &topic
	return bm.refreshTopics(tmp)
}
<commit_msg>tryLeader function implemented<commit_after>package kafka

import "sync"

type topicPartition struct {
	topic     string
	partition int32
}

type brokerManager struct {
	client        *Client
	defaultBroker *broker

	brokers     map[int32]*broker
	leaders     map[topicPartition]int32
	brokersLock sync.RWMutex
}

func newBrokerManager(client *Client, host string, port int32) (bm *brokerManager, err error) {
	bm = new(brokerManager)

	bm.client = client

	// we create a new broker object as the default 'master' broker
	// if this broker is also a leader then we will end up with two broker objects for it, but that's not a big deal
	bm.defaultBroker, err = newBroker(host, port)
	if err != nil {
		return nil, err
	}

	bm.brokers = make(map[int32]*broker)
	bm.leaders = make(map[topicPartition]int32)

	// do an initial fetch of all cluster metadata by specifing an empty list of topics
	err = bm.refreshTopics(make([]*string, 0))
	if err != nil {
		return nil, err
	}

	return bm, nil
}

func (bm *brokerManager) getLeader(topic string, partition int32) (*broker, error) {
	var broker *broker = nil
	bm.brokersLock.RLock()
	id, ok := bm.leaders[topicPartition{topic, partition}]
	if ok {
		broker = bm.brokers[id]
	}
	bm.brokersLock.RUnlock()

	if broker == nil {
		err := bm.refreshTopic(topic)
		if err != nil {
			return nil, err
		}
		bm.brokersLock.RLock()
		broker = bm.brokers[bm.leaders[topicPartition{topic, partition}]]
		bm.brokersLock.RUnlock()
	}

	if broker == nil {
		return nil, UNKNOWN_TOPIC_OR_PARTITION
	}

	return broker, nil
}

func (bm *brokerManager) tryLeader(topic string, partition int32, req encoder, res decoder) error {
	b, err := bm.getLeader(topic, partition)
	if err != nil {
		return err
	}

	responseChan, err := b.sendRequest(bm.client.id, req)
	if err != nil {
		// errors that would make us refresh the broker metadata don't get returned here,
		// they'd come through responseChan.errors, so it's safe to just return here
		return err
	}

	select {
	case buf := <-responseChan.packets:
		decoder := realDecoder{raw: buf}
		err = res.decode(&decoder)
	case err = <-responseChan.errors:
	}

	if err == nil {
		// successfully received and decoded the packet, we're done
		return nil
	}

	// we got an error, so discard that broker
	bm.brokersLock.Lock()
	delete(bm.brokers, b.id)
	bm.brokersLock.Unlock()

	// then do the whole thing again treating all errors as fatal
	// (the metadata for the broker gets refreshed automatically in getLeader)
	b, err = bm.getLeader(topic, partition)
	if err != nil {
		return err
	}

	responseChan, err = b.sendRequest(bm.client.id, req)
	if err != nil {
		return err
	}

	select {
	case buf := <-responseChan.packets:
		decoder := realDecoder{raw: buf}
		err = res.decode(&decoder)
		return err
	case err = <-responseChan.errors:
		return err
	}

}

func (bm *brokerManager) getDefault() *broker {

	if bm.defaultBroker == nil {
		bm.brokersLock.RLock()
		defer bm.brokersLock.RUnlock()
		for _, id := range bm.leaders {
			bm.defaultBroker = bm.brokers[id]
			break
		}
	}

	return bm.defaultBroker
}

func (bm *brokerManager) tryDefaultBrokers(req encoder, res decoder) error {
	for b := bm.getDefault(); b != nil; b = bm.getDefault() {
		responseChan, err := b.sendRequest(bm.client.id, req)
		if err != nil {
			return err
		}

		select {
		case buf := <-responseChan.packets:
			decoder := realDecoder{raw: buf}
			err = res.decode(&decoder)
			return err
		case <-responseChan.errors:
			bm.defaultBroker = nil
			bm.brokersLock.Lock()
			delete(bm.brokers, b.id)
			bm.brokersLock.Unlock()
		}
	}
	return OutOfBrokers{}
}

func (bm *brokerManager) refreshTopics(topics []*string) error {
	response := new(metadata)
	err := bm.tryDefaultBrokers(&metadataRequest{topics}, response)
	if err != nil {
		return err
	}

	bm.brokersLock.Lock()
	defer bm.brokersLock.Unlock()

	for i := range response.brokers {
		broker := &response.brokers[i]
		bm.brokers[broker.id] = broker
	}

	for i := range response.topics {
		topic := &response.topics[i]
		if topic.err != NO_ERROR {
			return topic.err
		}
		for j := range topic.partitions {
			partition := &topic.partitions[j]
			if partition.err != NO_ERROR {
				return partition.err
			}
			bm.leaders[topicPartition{*topic.name, partition.id}] = partition.leader
		}
	}

	return nil
}

func (bm *brokerManager) refreshTopic(topic string) error {
	tmp := make([]*string, 1)
	tmp[0] = &topic
	return bm.refreshTopics(tmp)
}
<|endoftext|>


<commit_before>package signal

import (
	"context"
	"time"
)

type ActivityTimer struct {
	updated chan bool
	timeout time.Duration
	ctx     context.Context
	cancel  context.CancelFunc
}

func (t *ActivityTimer) UpdateActivity() {
	select {
	case t.updated <- true:
	default:
	}
}

func (t *ActivityTimer) run() {
	for {
		select {
		case <-time.After(t.timeout):
			t.cancel()
			return
		case <-t.ctx.Done():
			return
		default:
			select {
			case <-time.After(t.timeout):
				t.cancel()
				return
			case <-t.ctx.Done():
				return
			case <-t.updated:
			}
		}
	}
}

func CancelAfterInactivity(ctx context.Context, cancel context.CancelFunc, timeout time.Duration) *ActivityTimer {
	timer := &ActivityTimer{
		ctx:     ctx,
		cancel:  cancel,
		timeout: timeout,
		updated: make(chan bool, 1),
	}
	go timer.run()
	return timer
}
<commit_msg>update wait logic<commit_after>package signal

import (
	"context"
	"time"
)

type ActivityTimer struct {
	updated chan bool
	timeout time.Duration
	ctx     context.Context
	cancel  context.CancelFunc
}

func (t *ActivityTimer) UpdateActivity() {
	select {
	case t.updated <- true:
	default:
	}
}

func (t *ActivityTimer) run() {
	for {
		select {
		case <-time.After(t.timeout):
		case <-t.ctx.Done():
			return
		}
		select {
		case <-t.updated:
		// Updated keep waiting.
		default:
			t.cancel()
			return
		}

	}
}

func CancelAfterInactivity(ctx context.Context, cancel context.CancelFunc, timeout time.Duration) *ActivityTimer {
	timer := &ActivityTimer{
		ctx:     ctx,
		cancel:  cancel,
		timeout: timeout,
		updated: make(chan bool, 1),
	}
	go timer.run()
	return timer
}
<|endoftext|>


<commit_before>package gorocksdb

// #include "rocksdb/c.h"
import "C"
import "unsafe"

// An application can issue a read request (via Get/Iterators) and specify
// if that read should process data that ALREADY resides on a specified cache
// level. For example, if an application specifies BlockCacheTier then the
// Get call will process data that is already processed in the memtable or
// the block cache. It will not page in data from the OS cache or data that
// resides in storage.
type ReadTier uint

const (
	// data in memtable, block cache, OS cache or storage
	ReadAllTier = ReadTier(0)
	// data in memtable or block cache
	BlockCacheTier = ReadTier(1)
)

// ReadOptions represent all of the available options when reading from a
// database.
type ReadOptions struct {
	c *C.rocksdb_readoptions_t
}

// NewDefaultReadOptions creates a default ReadOptions object.
func NewDefaultReadOptions() *ReadOptions {
	return NewNativeReadOptions(C.rocksdb_readoptions_create())
}

// NewNativeReadOptions creates a ReadOptions object.
func NewNativeReadOptions(c *C.rocksdb_readoptions_t) *ReadOptions {
	return &ReadOptions{c}
}

// UnsafeGetReadOptions returns the underlying c read options object.
func (self *ReadOptions) UnsafeGetReadOptions() unsafe.Pointer {
	return unsafe.Pointer(self.c)
}

// If true, all data read from underlying storage will be
// verified against corresponding checksums.
// Default: false
func (self *ReadOptions) SetVerifyChecksums(value bool) {
	C.rocksdb_readoptions_set_verify_checksums(self.c, boolToChar(value))
}

// Should the "data block"/"index block"/"filter block" read for this
// iteration be cached in memory?
// Callers may wish to set this field to false for bulk scans.
// Default: true
func (self *ReadOptions) SetFillCache(value bool) {
	C.rocksdb_readoptions_set_fill_cache(self.c, boolToChar(value))
}

// If snapshot is set, read as of the supplied snapshot
// which must belong to the DB that is being read and which must
// not have been released.
// Default: nil
func (self *ReadOptions) SetSnapshot(snap *Snapshot) {
	C.rocksdb_readoptions_set_snapshot(self.c, snap.c)
}

// Specify if this read request should process data that ALREADY
// resides on a particular cache. If the required data is not
// found at the specified cache, then Status::Incomplete is returned.
// Default: ReadAllTier
func (self *ReadOptions) SetReadTier(value ReadTier) {
	C.rocksdb_readoptions_set_read_tier(self.c, C.int(value))
}

// Specify to create a tailing iterator -- a special iterator that has a
// view of the complete database (i.e. it can also be used to read newly
// added data) and is optimized for sequential reads. It will return records
// that were inserted into the database after the creation of the iterator.
// Default: false
func (self *ReadOptions) SetTailing(value bool) {
	C.rocksdb_readoptions_set_tailing(self.c, boolToChar(value))
}

// Destroy deallocates the ReadOptions object.
func (self *ReadOptions) Destroy() {
	C.rocksdb_readoptions_destroy(self.c)
	self.c = nil
}
<commit_msg>references set total order mode support for read options<commit_after>package gorocksdb

// #include "rocksdb/c.h"
import "C"
import "unsafe"

// An application can issue a read request (via Get/Iterators) and specify
// if that read should process data that ALREADY resides on a specified cache
// level. For example, if an application specifies BlockCacheTier then the
// Get call will process data that is already processed in the memtable or
// the block cache. It will not page in data from the OS cache or data that
// resides in storage.
type ReadTier uint

const (
	// data in memtable, block cache, OS cache or storage
	ReadAllTier = ReadTier(0)
	// data in memtable or block cache
	BlockCacheTier = ReadTier(1)
)

// ReadOptions represent all of the available options when reading from a
// database.
type ReadOptions struct {
	c *C.rocksdb_readoptions_t
}

// NewDefaultReadOptions creates a default ReadOptions object.
func NewDefaultReadOptions() *ReadOptions {
	return NewNativeReadOptions(C.rocksdb_readoptions_create())
}

// NewNativeReadOptions creates a ReadOptions object.
func NewNativeReadOptions(c *C.rocksdb_readoptions_t) *ReadOptions {
	return &ReadOptions{c}
}

// UnsafeGetReadOptions returns the underlying c read options object.
func (self *ReadOptions) UnsafeGetReadOptions() unsafe.Pointer {
	return unsafe.Pointer(self.c)
}

// If true, all data read from underlying storage will be
// verified against corresponding checksums.
// Default: false
func (self *ReadOptions) SetVerifyChecksums(value bool) {
	C.rocksdb_readoptions_set_verify_checksums(self.c, boolToChar(value))
}

// Should the "data block"/"index block"/"filter block" read for this
// iteration be cached in memory?
// Callers may wish to set this field to false for bulk scans.
// Default: true
func (self *ReadOptions) SetFillCache(value bool) {
	C.rocksdb_readoptions_set_fill_cache(self.c, boolToChar(value))
}

// If snapshot is set, read as of the supplied snapshot
// which must belong to the DB that is being read and which must
// not have been released.
// Default: nil
func (self *ReadOptions) SetSnapshot(snap *Snapshot) {
	C.rocksdb_readoptions_set_snapshot(self.c, snap.c)
}

// Specify if this read request should process data that ALREADY
// resides on a particular cache. If the required data is not
// found at the specified cache, then Status::Incomplete is returned.
// Default: ReadAllTier
func (self *ReadOptions) SetReadTier(value ReadTier) {
	C.rocksdb_readoptions_set_read_tier(self.c, C.int(value))
}

// Specify to create a tailing iterator -- a special iterator that has a
// view of the complete database (i.e. it can also be used to read newly
// added data) and is optimized for sequential reads. It will return records
// that were inserted into the database after the creation of the iterator.
// Default: false
func (self *ReadOptions) SetTailing(value bool) {
	C.rocksdb_readoptions_set_tailing(self.c, boolToChar(value))
}

func (self *ReadOptions) SetTotalOrderMode(value bool) {
	C.rocksdb_readoptions_set_total_order_mode(self.c, boolToChar(value))
}

// Destroy deallocates the ReadOptions object.
func (self *ReadOptions) Destroy() {
	C.rocksdb_readoptions_destroy(self.c)
	self.c = nil
}
<|endoftext|>


<commit_before>package wrapper

import (
	"context"
	"errors"
	"fmt"

	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"
	"github.com/grafana/grafana_plugin_model/go/datasource"
)

func NewDatasourcePluginWrapper(log log.Logger, plugin datasource.DatasourcePlugin) *DatasourcePluginWrapper {
	return &DatasourcePluginWrapper{DatasourcePlugin: plugin, logger: log}
}

type DatasourcePluginWrapper struct {
	datasource.DatasourcePlugin
	logger log.Logger
}

func (tw *DatasourcePluginWrapper) Query(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
	jsonData, err := ds.JsonData.MarshalJSON()
	if err != nil {
		return nil, err
	}

	pbQuery := &datasource.DatasourceRequest{
		Datasource: &datasource.DatasourceInfo{
			Name:                    ds.Name,
			Type:                    ds.Type,
			Url:                     ds.Url,
			Id:                      ds.Id,
			OrgId:                   ds.OrgId,
			JsonData:                string(jsonData),
			DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
		},
		TimeRange: &datasource.TimeRange{
			FromRaw:     query.TimeRange.From,
			ToRaw:       query.TimeRange.To,
			ToEpochMs:   query.TimeRange.GetToAsMsEpoch(),
			FromEpochMs: query.TimeRange.GetFromAsMsEpoch(),
		},
		Queries: []*datasource.Query{},
	}

	for _, q := range query.Queries {
		modelJson, _ := q.Model.MarshalJSON()

		pbQuery.Queries = append(pbQuery.Queries, &datasource.Query{
			ModelJson:     string(modelJson),
			IntervalMs:    q.IntervalMs,
			RefId:         q.RefId,
			MaxDataPoints: q.MaxDataPoints,
		})
	}

	pbres, err := tw.DatasourcePlugin.Query(ctx, pbQuery)

	if err != nil {
		return nil, err
	}

	res := &tsdb.Response{
		Results: map[string]*tsdb.QueryResult{},
	}

	for _, r := range pbres.Results {
		qr := &tsdb.QueryResult{
			RefId:  r.RefId,
			Series: []*tsdb.TimeSeries{},
			Tables: []*tsdb.Table{},
		}

		if r.Error != "" {
			qr.Error = errors.New(r.Error)
			qr.ErrorString = r.Error
		}

		if r.MetaJson != "" {
			metaJson, _ := simplejson.NewJson([]byte(r.MetaJson))
			qr.Meta = metaJson
		}

		for _, s := range r.GetSeries() {
			points := tsdb.TimeSeriesPoints{}

			for _, p := range s.Points {
				po := tsdb.NewTimePoint(null.FloatFrom(p.Value), float64(p.Timestamp))
				points = append(points, po)
			}

			qr.Series = append(qr.Series, &tsdb.TimeSeries{
				Name:   s.Name,
				Tags:   s.Tags,
				Points: points,
			})
		}

		mappedTables, err := tw.mapTables(r)
		if err != nil {
			return nil, err
		}
		qr.Tables = mappedTables

		res.Results[r.RefId] = qr
	}

	return res, nil
}
func (tw *DatasourcePluginWrapper) mapTables(r *datasource.QueryResult) ([]*tsdb.Table, error) {
	var tables []*tsdb.Table
	for _, t := range r.GetTables() {
		mappedTable, err := tw.mapTable(t)
		if err != nil {
			return nil, err
		}
		tables = append(tables, mappedTable)
	}
	return tables, nil
}

func (tw *DatasourcePluginWrapper) mapTable(t *datasource.Table) (*tsdb.Table, error) {
	table := &tsdb.Table{}
	for _, c := range t.GetColumns() {
		table.Columns = append(table.Columns, tsdb.TableColumn{
			Text: c.Name,
		})
	}

	table.Rows = make([]tsdb.RowValues, 0)
	for _, r := range t.GetRows() {
		row := tsdb.RowValues{}
		for _, rv := range r.Values {
			mappedRw, err := tw.mapRowValue(rv)
			if err != nil {
				return nil, err
			}

			row = append(row, mappedRw)
		}
		table.Rows = append(table.Rows, row)
	}

	return table, nil
}
func (tw *DatasourcePluginWrapper) mapRowValue(rv *datasource.RowValue) (interface{}, error) {
	switch rv.Kind {
	case datasource.RowValue_TYPE_NULL:
		return nil, nil
	case datasource.RowValue_TYPE_INT64:
		return rv.Int64Value, nil
	case datasource.RowValue_TYPE_BOOL:
		return rv.BoolValue, nil
	case datasource.RowValue_TYPE_STRING:
		return rv.StringValue, nil
	case datasource.RowValue_TYPE_DOUBLE:
		return rv.DoubleValue, nil
	case datasource.RowValue_TYPE_BYTES:
		return rv.BytesValue, nil
	default:
		return nil, fmt.Errorf("Unsupported row value %v from plugin", rv.Kind)
	}
}
<commit_msg>backend plugins: log an error if parsing meta field failed<commit_after>package wrapper

import (
	"context"
	"errors"
	"fmt"

	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"
	"github.com/grafana/grafana_plugin_model/go/datasource"
)

func NewDatasourcePluginWrapper(log log.Logger, plugin datasource.DatasourcePlugin) *DatasourcePluginWrapper {
	return &DatasourcePluginWrapper{DatasourcePlugin: plugin, logger: log}
}

type DatasourcePluginWrapper struct {
	datasource.DatasourcePlugin
	logger log.Logger
}

func (tw *DatasourcePluginWrapper) Query(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
	jsonData, err := ds.JsonData.MarshalJSON()
	if err != nil {
		return nil, err
	}

	pbQuery := &datasource.DatasourceRequest{
		Datasource: &datasource.DatasourceInfo{
			Name:                    ds.Name,
			Type:                    ds.Type,
			Url:                     ds.Url,
			Id:                      ds.Id,
			OrgId:                   ds.OrgId,
			JsonData:                string(jsonData),
			DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
		},
		TimeRange: &datasource.TimeRange{
			FromRaw:     query.TimeRange.From,
			ToRaw:       query.TimeRange.To,
			ToEpochMs:   query.TimeRange.GetToAsMsEpoch(),
			FromEpochMs: query.TimeRange.GetFromAsMsEpoch(),
		},
		Queries: []*datasource.Query{},
	}

	for _, q := range query.Queries {
		modelJson, _ := q.Model.MarshalJSON()

		pbQuery.Queries = append(pbQuery.Queries, &datasource.Query{
			ModelJson:     string(modelJson),
			IntervalMs:    q.IntervalMs,
			RefId:         q.RefId,
			MaxDataPoints: q.MaxDataPoints,
		})
	}

	pbres, err := tw.DatasourcePlugin.Query(ctx, pbQuery)

	if err != nil {
		return nil, err
	}

	res := &tsdb.Response{
		Results: map[string]*tsdb.QueryResult{},
	}

	for _, r := range pbres.Results {
		qr := &tsdb.QueryResult{
			RefId:  r.RefId,
			Series: []*tsdb.TimeSeries{},
			Tables: []*tsdb.Table{},
		}

		if r.Error != "" {
			qr.Error = errors.New(r.Error)
			qr.ErrorString = r.Error
		}

		if r.MetaJson != "" {
			metaJson, err := simplejson.NewJson([]byte(r.MetaJson))
			if err != nil {
				tw.logger.Error("Error parsing JSON Meta field: " + err.Error())
			}
			qr.Meta = metaJson
		}

		for _, s := range r.GetSeries() {
			points := tsdb.TimeSeriesPoints{}

			for _, p := range s.Points {
				po := tsdb.NewTimePoint(null.FloatFrom(p.Value), float64(p.Timestamp))
				points = append(points, po)
			}

			qr.Series = append(qr.Series, &tsdb.TimeSeries{
				Name:   s.Name,
				Tags:   s.Tags,
				Points: points,
			})
		}

		mappedTables, err := tw.mapTables(r)
		if err != nil {
			return nil, err
		}
		qr.Tables = mappedTables

		res.Results[r.RefId] = qr
	}

	return res, nil
}
func (tw *DatasourcePluginWrapper) mapTables(r *datasource.QueryResult) ([]*tsdb.Table, error) {
	var tables []*tsdb.Table
	for _, t := range r.GetTables() {
		mappedTable, err := tw.mapTable(t)
		if err != nil {
			return nil, err
		}
		tables = append(tables, mappedTable)
	}
	return tables, nil
}

func (tw *DatasourcePluginWrapper) mapTable(t *datasource.Table) (*tsdb.Table, error) {
	table := &tsdb.Table{}
	for _, c := range t.GetColumns() {
		table.Columns = append(table.Columns, tsdb.TableColumn{
			Text: c.Name,
		})
	}

	table.Rows = make([]tsdb.RowValues, 0)
	for _, r := range t.GetRows() {
		row := tsdb.RowValues{}
		for _, rv := range r.Values {
			mappedRw, err := tw.mapRowValue(rv)
			if err != nil {
				return nil, err
			}

			row = append(row, mappedRw)
		}
		table.Rows = append(table.Rows, row)
	}

	return table, nil
}
func (tw *DatasourcePluginWrapper) mapRowValue(rv *datasource.RowValue) (interface{}, error) {
	switch rv.Kind {
	case datasource.RowValue_TYPE_NULL:
		return nil, nil
	case datasource.RowValue_TYPE_INT64:
		return rv.Int64Value, nil
	case datasource.RowValue_TYPE_BOOL:
		return rv.BoolValue, nil
	case datasource.RowValue_TYPE_STRING:
		return rv.StringValue, nil
	case datasource.RowValue_TYPE_DOUBLE:
		return rv.DoubleValue, nil
	case datasource.RowValue_TYPE_BYTES:
		return rv.BytesValue, nil
	default:
		return nil, fmt.Errorf("Unsupported row value %v from plugin", rv.Kind)
	}
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package config

import (
	"reflect"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/log"
	"github.com/admpub/nging/v3/application/library/common"
	"github.com/admpub/nging/v3/application/library/notice"
	"github.com/admpub/nging/v3/application/registry/settings"
)

var Setting = common.Setting

func NewSettings(config *Config) *Settings {
	c := &Settings{
		config: config,
	}
	return c
}

type Settings struct {
	Email              Email  `json:"email"`
	Log                Log    `json:"log"`
	APIKey             string `json:"-"` //API密钥
	Debug              bool   `json:"debug"`
	MaxRequestBodySize int    `json:"maxRequestBodySize"`
	Base               echo.H `json:"base"`
	config             *Config
}

func (c *Settings) SetBy(r echo.H, defaults echo.H) *Settings {
	if !r.Has(`base`) && defaults != nil {
		r.Set(`base`, defaults.GetStore(`base`))
	}
	c.Base = r.GetStore(`base`)
	c.APIKey = c.Base.String(`apiKey`)
	c.Debug = c.Base.Bool(`debug`)
	c.MaxRequestBodySize = c.Base.Int(`maxRequestBodySize`)
	return c
}

func (c *Settings) SetDebug(on bool) {
	c.Log.Debug = on
	c.Debug = on

	c.config.DB.SetDebug(on)
	notice.SetDebug(on)
	if on {
		log.Info(`Currently running in debug mode`)
		log.SetLevel(`Debug`)
	} else {
		log.Info(`Currently running in normal mode`)
		log.SetLevel(`Info`)
	}
	subdomains.Default.SetDebug(on)
}

var (
	actGroups          = []string{`base`, `smtp`, `log`}
	onKeySetSettings   = map[string][]func(Diff) error{}
	onGroupSetSettings = map[string][]func(Diffs) error{}
)

func OnGroupSetSettings(groupAndKey string, fn func(Diffs) error) {
	if _, ok := onGroupSetSettings[groupAndKey]; !ok {
		onGroupSetSettings[groupAndKey] = []func(Diffs) error{}
	}
	onGroupSetSettings[groupAndKey] = append(onGroupSetSettings[groupAndKey], fn)
}

func OnKeySetSettings(groupAndKey string, fn func(Diff) error) {
	if _, ok := onKeySetSettings[groupAndKey]; !ok {
		onKeySetSettings[groupAndKey] = []func(Diff) error{}
	}
	onKeySetSettings[groupAndKey] = append(onKeySetSettings[groupAndKey], fn)
}

func FireInitSettings(configs echo.H) error {
	for group, fnList := range onGroupSetSettings {
		values := configs.GetStore(group)
		diffs := Diffs{}
		for k, v := range values {
			diffs[k] = &Diff{
				Old: v,
				New: v,
			}
		}
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for groupAndKey, fnList := range onKeySetSettings {
		args := strings.SplitN(groupAndKey, `.`, 2)
		values := configs.GetStore(args[0])
		var val interface{}
		if len(args) == 2 {
			val = values.Get(args[1])
		} else {
			val = values
		}
		for _, fn := range fnList {
			if err := fn(Diff{
				Old: val,
				New: val,
			}); err != nil {
				return err
			}
		}
	}
	return nil
}

func FireSetSettings(group string, diffs Diffs) error {
	if fnList, ok := onGroupSetSettings[group]; ok {
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for key, diff := range diffs {
		k := group + `.` + key
		if fnList, ok := onKeySetSettings[k]; ok {
			for _, fn := range fnList {
				if err := fn(*diff); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (c *Settings) Init() {
	defaults := settings.ConfigDefaultsAsStore()
	var configs = defaults
	if IsInstalled() {
		if c.config.ConnectedDB() {
			configs = settings.ConfigAsStore()
		}
	}
	echo.Set(`NgingConfig`, configs)
	for _, group := range actGroups {
		c.SetConfig(group, configs, defaults)
	}
	err := FireInitSettings(configs)
	if err != nil {
		log.Error(err)
	}
}

func (c *Settings) GetConfig() echo.H {
	r, _ := echo.Get(`NgingConfig`).(echo.H)
	return r
}

type Diff struct {
	Old interface{}
	New interface{}
}

func (d Diff) String() string {
	s, _ := d.New.(string)
	return s
}

type Diffs map[string]*Diff

func (d Diffs) Get(key string) interface{} {
	return d[key]
}

func (c *Settings) SetConfigs(groups ...string) {
	newConfigs := settings.ConfigAsStore(groups...)
	oldConfigs := c.GetConfig()
	c.setConfigs(newConfigs, oldConfigs)
}

func (c *Settings) setConfigs(newConfigs echo.H, oldConfigs echo.H) {
	for group, conf := range newConfigs {
		keyCfg := conf.(echo.H)
		keyOldCfg := oldConfigs.GetStore(group)
		diffs := Diffs{}
		for k, v := range keyCfg {
			if !reflect.DeepEqual(keyOldCfg.Get(k), v) {
				diffs[k] = &Diff{
					Old: keyOldCfg.Get(k),
					New: v,
				}
			}
		}
		if len(diffs) == 0 {
			continue
		}
		oldConfigs.Set(group, keyCfg)
		FireSetSettings(group, diffs)
		//log.Debug(`Change configuration:`, group, `:`, echo.Dump(conf, false))
		c.SetConfig(group, oldConfigs, nil)
	}
}

func (c *Settings) SetConfig(group string, ngingConfig echo.H, defaults echo.H) {
	switch group {
	case `base`:
		c.SetBy(ngingConfig, defaults)
		c.SetDebug(c.Debug)
	case `smtp`:
		c.Email.SetBy(ngingConfig, defaults).Init()
	case `log`:
		c.Log.SetBy(ngingConfig, defaults).Init()
	}
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package config

import (
	"reflect"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/log"
	"github.com/admpub/nging/v3/application/library/common"
	"github.com/admpub/nging/v3/application/library/notice"
	"github.com/admpub/nging/v3/application/registry/settings"
)

var Setting = common.Setting

func NewSettings(config *Config) *Settings {
	c := &Settings{
		config: config,
	}
	return c
}

type Settings struct {
	Email              Email  `json:"email"`
	Log                Log    `json:"log"`
	APIKey             string `json:"-"` //API密钥
	Debug              bool   `json:"debug"`
	MaxRequestBodySize int    `json:"maxRequestBodySize"`
	Base               echo.H `json:"base"`
	config             *Config
}

func (c *Settings) SetBy(r echo.H, defaults echo.H) *Settings {
	if !r.Has(`base`) && defaults != nil {
		r.Set(`base`, defaults.GetStore(`base`))
	}
	c.Base = r.GetStore(`base`)
	c.APIKey = c.Base.String(`apiKey`)
	c.Debug = c.Base.Bool(`debug`)
	c.MaxRequestBodySize = c.Base.Int(`maxRequestBodySize`)
	return c
}

func (c *Settings) SetDebug(on bool) {
	c.Log.Debug = on
	c.Debug = on

	c.config.DB.SetDebug(on)
	notice.SetDebug(on)
	if on {
		log.Info(`Currently running in debug mode`)
		log.SetLevel(`Debug`)
	} else {
		log.Info(`Currently running in normal mode`)
		log.SetLevel(`Info`)
	}
	subdomains.Default.SetDebug(on)
}

var (
	actGroups          = []string{`base`, `smtp`, `log`}
	onKeySetSettings   = map[string][]func(Diff) error{}
	onGroupSetSettings = map[string][]func(Diffs) error{}
)

func OnGroupSetSettings(groupAndKey string, fn func(Diffs) error) {
	if _, ok := onGroupSetSettings[groupAndKey]; !ok {
		onGroupSetSettings[groupAndKey] = []func(Diffs) error{}
	}
	onGroupSetSettings[groupAndKey] = append(onGroupSetSettings[groupAndKey], fn)
}

func OnKeySetSettings(groupAndKey string, fn func(Diff) error) {
	if _, ok := onKeySetSettings[groupAndKey]; !ok {
		onKeySetSettings[groupAndKey] = []func(Diff) error{}
	}
	onKeySetSettings[groupAndKey] = append(onKeySetSettings[groupAndKey], fn)
}

func FireInitSettings(configs echo.H) error {
	for group, fnList := range onGroupSetSettings {
		values := configs.GetStore(group)
		diffs := Diffs{}
		for k, v := range values {
			diffs[k] = &Diff{
				Old: v,
				New: v,
			}
		}
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for groupAndKey, fnList := range onKeySetSettings {
		args := strings.SplitN(groupAndKey, `.`, 2)
		values := configs.GetStore(args[0])
		var val interface{}
		if len(args) == 2 {
			val = values.Get(args[1])
		} else {
			val = values
		}
		for _, fn := range fnList {
			if err := fn(Diff{
				Old: val,
				New: val,
			}); err != nil {
				return err
			}
		}
	}
	return nil
}

func FireSetSettings(group string, diffs Diffs) error {
	if fnList, ok := onGroupSetSettings[group]; ok {
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for key, diff := range diffs {
		k := group + `.` + key
		if fnList, ok := onKeySetSettings[k]; ok {
			for _, fn := range fnList {
				if err := fn(*diff); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (c *Settings) Init() {
	defaults := settings.ConfigDefaultsAsStore()
	var configs = defaults
	if IsInstalled() {
		if c.config.ConnectedDB() {
			configs = settings.ConfigAsStore()
		}
	}
	echo.Set(`NgingConfig`, configs)
	for _, group := range actGroups {
		c.SetConfig(group, configs, defaults)
	}
	err := FireInitSettings(configs)
	if err != nil {
		log.Error(err)
	}
}

func (c *Settings) GetConfig() echo.H {
	r, _ := echo.Get(`NgingConfig`).(echo.H)
	return r
}

type Diff struct {
	Old    interface{}
	New    interface{}
	IsDiff bool
}

func (d Diff) String() string {
	s, _ := d.New.(string)
	return s
}

type Diffs map[string]*Diff

func (d Diffs) Get(key string) interface{} {
	return d[key]
}

func (c *Settings) SetConfigs(groups ...string) {
	newConfigs := settings.ConfigAsStore(groups...)
	oldConfigs := c.GetConfig()
	c.setConfigs(newConfigs, oldConfigs)
}

func (c *Settings) setConfigs(newConfigs echo.H, oldConfigs echo.H) {
	for group, conf := range newConfigs {
		keyCfg := conf.(echo.H)
		keyOldCfg := oldConfigs.GetStore(group)
		diffs := Diffs{}
		for k, v := range keyCfg {
			if !reflect.DeepEqual(keyOldCfg.Get(k), v) {
				diffs[k] = &Diff{
					Old:    keyOldCfg.Get(k),
					New:    v,
					IsDiff: true,
				}
			}
		}
		if len(diffs) == 0 {
			continue
		}
		oldConfigs.Set(group, keyCfg)
		FireSetSettings(group, diffs)
		//log.Debug(`Change configuration:`, group, `:`, echo.Dump(conf, false))
		c.SetConfig(group, oldConfigs, nil)
	}
}

func (c *Settings) SetConfig(group string, ngingConfig echo.H, defaults echo.H) {
	switch group {
	case `base`:
		c.SetBy(ngingConfig, defaults)
		c.SetDebug(c.Debug)
	case `smtp`:
		c.Email.SetBy(ngingConfig, defaults).Init()
	case `log`:
		c.Log.SetBy(ngingConfig, defaults).Init()
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSesNotification() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSesNotificationSet,
		Read:   resourceAwsSesNotificationRead,
		Update: resourceAwsSesNotificationSet,
		Delete: resourceAwsSesNotificationDelete,

		Schema: map[string]*schema.Schema{
			"topic_arn": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateArn,
			},

			"notification_type": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: validation.StringInSlice([]string{
					ses.NotificationTypeBounce,
					ses.NotificationTypeComplaint,
					ses.NotificationTypeDelivery,
				}, false),
			},

			"identity": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.NoZeroValues,
			},
		},
	}
}

func resourceAwsSesNotificationSet(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn
	topic := d.Get("topic_arn").(string)
	notification := d.Get("notification_type").(string)
	identity := d.Get("identity").(string)

	setOpts := &ses.SetIdentityNotificationTopicInput{
		Identity:         aws.String(identity),
		NotificationType: aws.String(notification),
		SnsTopic:         aws.String(topic),
	}

	d.SetId(fmt.Sprintf("%s|%s", identity, notification))

	log.Printf("[DEBUG] Setting SES Identity Notification: %#v", setOpts)

	if _, err := conn.SetIdentityNotificationTopic(setOpts); err != nil {
		return fmt.Errorf("Error setting SES Identity Notification: %s", err)
	}

	return resourceAwsSesNotificationRead(d, meta)
}

func resourceAwsSesNotificationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	identity, notificationType, err := decodeSesIdentityNotificationId(d.Id())
	if err != nil {
		return err
	}

	getOpts := &ses.GetIdentityNotificationAttributesInput{
		Identities: []*string{aws.String(identity)},
	}

	log.Printf("[DEBUG] Reading SES Identity Notification Attributes: %#v", getOpts)

	response, err := conn.GetIdentityNotificationAttributes(getOpts)

	if err != nil {
		return fmt.Errorf("Error reading SES Identity Notification: %s", err)
	}

	notificationAttributes := response.NotificationAttributes[identity]
	switch notificationType {
	case ses.NotificationTypeBounce:
		d.Set("topic_arn", notificationAttributes.BounceTopic)
	case ses.NotificationTypeComplaint:
		d.Set("topic_arn", notificationAttributes.ComplaintTopic)
	case ses.NotificationTypeDelivery:
		d.Set("topic_arn", notificationAttributes.DeliveryTopic)
	}

	return nil
}

func resourceAwsSesNotificationDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	identity, notificationType, err := decodeSesIdentityNotificationId(d.Id())
	if err != nil {
		return err
	}

	setOpts := &ses.SetIdentityNotificationTopicInput{
		Identity:         aws.String(identity),
		NotificationType: aws.String(notification),
		SnsTopic:         nil,
	}

	log.Printf("[DEBUG] Deleting SES Identity Notification: %#v", setOpts)

	if _, err := conn.SetIdentityNotificationTopic(setOpts); err != nil {
		return fmt.Errorf("Error deleting SES Identity Notification: %s", err)
	}

	return resourceAwsSesNotificationRead(d, meta)
}

func decodeSesIdentityNotificationId(id string) (string, string, error) {
	parts := strings.Split(id, "|")
	if len(parts) != 2 {
		return "", "", fmt.Errorf("Unexpected format of ID (%q), expected IDENTITY|TYPE", id)
	}
	return parts[0], parts[1], nil
}
<commit_msg>#931 changed variable<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSesNotification() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSesNotificationSet,
		Read:   resourceAwsSesNotificationRead,
		Update: resourceAwsSesNotificationSet,
		Delete: resourceAwsSesNotificationDelete,

		Schema: map[string]*schema.Schema{
			"topic_arn": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateArn,
			},

			"notification_type": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: validation.StringInSlice([]string{
					ses.NotificationTypeBounce,
					ses.NotificationTypeComplaint,
					ses.NotificationTypeDelivery,
				}, false),
			},

			"identity": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.NoZeroValues,
			},
		},
	}
}

func resourceAwsSesNotificationSet(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn
	topic := d.Get("topic_arn").(string)
	notification := d.Get("notification_type").(string)
	identity := d.Get("identity").(string)

	setOpts := &ses.SetIdentityNotificationTopicInput{
		Identity:         aws.String(identity),
		NotificationType: aws.String(notification),
		SnsTopic:         aws.String(topic),
	}

	d.SetId(fmt.Sprintf("%s|%s", identity, notification))

	log.Printf("[DEBUG] Setting SES Identity Notification: %#v", setOpts)

	if _, err := conn.SetIdentityNotificationTopic(setOpts); err != nil {
		return fmt.Errorf("Error setting SES Identity Notification: %s", err)
	}

	return resourceAwsSesNotificationRead(d, meta)
}

func resourceAwsSesNotificationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	identity, notificationType, err := decodeSesIdentityNotificationId(d.Id())
	if err != nil {
		return err
	}

	getOpts := &ses.GetIdentityNotificationAttributesInput{
		Identities: []*string{aws.String(identity)},
	}

	log.Printf("[DEBUG] Reading SES Identity Notification Attributes: %#v", getOpts)

	response, err := conn.GetIdentityNotificationAttributes(getOpts)

	if err != nil {
		return fmt.Errorf("Error reading SES Identity Notification: %s", err)
	}

	notificationAttributes := response.NotificationAttributes[identity]
	switch notificationType {
	case ses.NotificationTypeBounce:
		d.Set("topic_arn", notificationAttributes.BounceTopic)
	case ses.NotificationTypeComplaint:
		d.Set("topic_arn", notificationAttributes.ComplaintTopic)
	case ses.NotificationTypeDelivery:
		d.Set("topic_arn", notificationAttributes.DeliveryTopic)
	}

	return nil
}

func resourceAwsSesNotificationDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).sesConn

	identity, notificationType, err := decodeSesIdentityNotificationId(d.Id())
	if err != nil {
		return err
	}

	setOpts := &ses.SetIdentityNotificationTopicInput{
		Identity:         aws.String(identity),
		NotificationType: aws.String(notificationType),
		SnsTopic:         nil,
	}

	log.Printf("[DEBUG] Deleting SES Identity Notification: %#v", setOpts)

	if _, err := conn.SetIdentityNotificationTopic(setOpts); err != nil {
		return fmt.Errorf("Error deleting SES Identity Notification: %s", err)
	}

	return resourceAwsSesNotificationRead(d, meta)
}

func decodeSesIdentityNotificationId(id string) (string, string, error) {
	parts := strings.Split(id, "|")
	if len(parts) != 2 {
		return "", "", fmt.Errorf("Unexpected format of ID (%q), expected IDENTITY|TYPE", id)
	}
	return parts[0], parts[1], nil
}
<|endoftext|>


<commit_before>//
// templates.go
// template model interfaces
//
package srnd

import (
  "fmt"
  "github.com/hoisie/mustache"
  "io"
  "io/ioutil"
  "log"
  "path/filepath"
  "sort"
)

type templateEngine struct {
  // every newsgroup
  groups map[string]GroupModel
  // loaded templates
  templates map[string]string
  // root directory for templates
  template_dir string
}

func (self templateEngine) templateCached(name string) (ok bool) {
  _, ok = self.templates[name]
  return
}

func (self templateEngine) getTemplate(name string) (t string) {
  if self.templateCached(name) {
    t, _ = self.templates[name]
  } else {
    // ignores errors, this is probably bad
    b, _ := ioutil.ReadFile(filepath.Join(self.template_dir, name))
    t = string(b)
    self.templates[name] = t
  }
  return
}

func (self templateEngine) renderTemplate(name string, obj interface{}) string {
  t := self.getTemplate(name)
  return mustache.Render(t, obj)
}

// get a board model given a newsgroup
// load un updated board model if we don't have it
func (self templateEngine) obtainBoard(prefix, frontend, group string, db Database) (model GroupModel) {
  model, ok := self.groups[group]
  if ! ok  {
    p := db.GetGroupPageCount(group)
    pages := int(p)
    // ignore error
    perpage, _ := db.GetThreadsPerPage(group)
    for page := 0 ; page < pages ; page ++ {
      model = append(model, db.GetGroupForPage(prefix, frontend, group, page, int(perpage)))
    }
    self.groups[group] = model
  }
  return

}
// generate a board page
func (self templateEngine) genBoardPage(prefix, frontend, newsgroup string, page int, outfile string, db Database) {
  // get it
  board := self.obtainBoard(prefix, frontend, newsgroup, db)
  // update it
  board = board.Update(page, db)
  wr, err := OpenFileWriter(outfile)
  if err == nil {
    board[page].RenderTo(wr)
    wr.Close()
    log.Println("wrote file", outfile)
  } else {
    log.Println("error generating board page", page, "for", newsgroup, err)
  }
  // save it
  self.groups[newsgroup] = board
}

// generate every page for a board
func (self templateEngine) genBoard(prefix, frontend, newsgroup, outdir string, db Database) {
  // get it
  board := self.obtainBoard(prefix, frontend, newsgroup, db)
  // update it
  board = board.UpdateAll(db)
  // save it
  self.groups[newsgroup] = board

  pages := len(board)
  for page := 0 ; page < pages ; page ++ {
    outfile := filepath.Join(outdir, fmt.Sprintf("%s-%d.html", newsgroup, page))
    wr, err := OpenFileWriter(outfile)
    if err == nil {
      board[page].RenderTo(wr)
      wr.Close()
      log.Println("wrote file", outfile)
    } else {
      log.Println("error generating board page", page, "for", newsgroup, err)
    }
  }
}

func (self templateEngine) genUkko(prefix, frontend, outfile string, database Database) {
  // get the last 15 bumped threads globally
  var threads []ThreadModel
  for _, article := range database.GetLastBumpedThreads("", 15) {
    newsgroup, msgid := article[0], article[1]
    // obtain board
    board := self.obtainBoard(prefix, frontend, newsgroup, database)
    board = board.Update(0, database)
    for _, th := range(board[0].Threads()) {
      if th.OP().MessageID() == msgid {
        threads = append(threads, th)
        break
      }
    }
    // save state of board
    self.groups[newsgroup] = board
  }
  wr, err := OpenFileWriter(outfile)
  if err == nil {
    io.WriteString(wr, template.renderTemplate("ukko.mustache", map[string]interface{} { "prefix" : prefix, "threads" : threads }))
    wr.Close()
    log.Println("wrote file", outfile)
  } else {
    log.Println("error generating ukko", err)
  }
}

func (self templateEngine) genThread(messageID, prefix, frontend, outfile string, db Database) {

  newsgroup, page, err := db.GetPageForRootMessage(messageID)
  if err != nil {
    log.Println(err)
  }
  // get it
  board := self.obtainBoard(prefix, frontend, newsgroup, db)
  // update our thread
  board[page] = board[page].Update(db)
  // save it
  self.groups[newsgroup] = board
  for _, th := range board[page].Threads() {
    if th.OP().MessageID() == messageID {
      // we found it
      wr, err := OpenFileWriter(outfile)
      if err == nil {
        th.RenderTo(wr)
        wr.Close()
        log.Println("wrote file", outfile)
      } else {
        log.Println(err)
      }
    }
  }
}

func newTemplateEngine(dir string) *templateEngine {
  return &templateEngine{
    groups: make(map[string]GroupModel),
    templates: make(map[string]string),
    template_dir: dir,
  }
}

var template = newTemplateEngine(defaultTemplateDir())

func renderPostForm(prefix, board, op_msg_id string) string {
  url := prefix + "post/" + board
  button := "New Thread"
  if op_msg_id != "" {
    button = "Reply"
  }
  return template.renderTemplate("postform.mustache", map[string]string { "post_url" : url, "reference" : op_msg_id , "button" : button } )
}

func (self templateEngine) genFrontPage(top_count int, frontend_name, outfile string, db Database) {
  // the graph for the front page
  var frontpage_graph boardPageRows

  // for each group
  groups := db.GetAllNewsgroups()
  for idx, group := range groups {
    if idx >= top_count {
      break
    }
    // posts per hour
    hour := db.CountPostsInGroup(group, 3600)
    // posts per day
    day := db.CountPostsInGroup(group, 86400)
    // posts total
    all := db.CountPostsInGroup(group, 0)
    frontpage_graph = append(frontpage_graph, boardPageRow{
      All: all,
      Day: day,
      Hour: hour,
      Board: group,
    })
  }
  wr, err := OpenFileWriter(outfile)
  if err != nil {
    log.Println("cannot render front page", err)
    return
  }

  param := make(map[string]interface{})
  sort.Sort(frontpage_graph)
  param["graph"] = frontpage_graph
  param["frontend"] = frontend_name
  param["totalposts"] = db.ArticleCount()
  _, err = io.WriteString(wr, self.renderTemplate("frontpage.mustache", param))
  if err != nil {
    log.Println("error writing front page", err)
  }
  wr.Close()
  log.Println("wrote file", outfile)
}
<commit_msg>optimize genthread<commit_after>//
// templates.go
// template model interfaces
//
package srnd

import (
  "fmt"
  "github.com/hoisie/mustache"
  "io"
  "io/ioutil"
  "log"
  "path/filepath"
  "sort"
)

type templateEngine struct {
  // every newsgroup
  groups map[string]GroupModel
  // loaded templates
  templates map[string]string
  // root directory for templates
  template_dir string
}

func (self templateEngine) templateCached(name string) (ok bool) {
  _, ok = self.templates[name]
  return
}

func (self templateEngine) getTemplate(name string) (t string) {
  if self.templateCached(name) {
    t, _ = self.templates[name]
  } else {
    // ignores errors, this is probably bad
    b, _ := ioutil.ReadFile(filepath.Join(self.template_dir, name))
    t = string(b)
    self.templates[name] = t
  }
  return
}

func (self templateEngine) renderTemplate(name string, obj interface{}) string {
  t := self.getTemplate(name)
  return mustache.Render(t, obj)
}

// get a board model given a newsgroup
// load un updated board model if we don't have it
func (self templateEngine) obtainBoard(prefix, frontend, group string, db Database) (model GroupModel) {
  model, ok := self.groups[group]
  if ! ok  {
    p := db.GetGroupPageCount(group)
    pages := int(p)
    // ignore error
    perpage, _ := db.GetThreadsPerPage(group)
    for page := 0 ; page < pages ; page ++ {
      model = append(model, db.GetGroupForPage(prefix, frontend, group, page, int(perpage)))
    }
    self.groups[group] = model
  }
  return

}
// generate a board page
func (self templateEngine) genBoardPage(prefix, frontend, newsgroup string, page int, outfile string, db Database) {
  // get it
  board := self.obtainBoard(prefix, frontend, newsgroup, db)
  // update it
  board = board.Update(page, db)
  wr, err := OpenFileWriter(outfile)
  if err == nil {
    board[page].RenderTo(wr)
    wr.Close()
    log.Println("wrote file", outfile)
  } else {
    log.Println("error generating board page", page, "for", newsgroup, err)
  }
  // save it
  self.groups[newsgroup] = board
}

// generate every page for a board
func (self templateEngine) genBoard(prefix, frontend, newsgroup, outdir string, db Database) {
  // get it
  board := self.obtainBoard(prefix, frontend, newsgroup, db)
  // update it
  board = board.UpdateAll(db)
  // save it
  self.groups[newsgroup] = board

  pages := len(board)
  for page := 0 ; page < pages ; page ++ {
    outfile := filepath.Join(outdir, fmt.Sprintf("%s-%d.html", newsgroup, page))
    wr, err := OpenFileWriter(outfile)
    if err == nil {
      board[page].RenderTo(wr)
      wr.Close()
      log.Println("wrote file", outfile)
    } else {
      log.Println("error generating board page", page, "for", newsgroup, err)
    }
  }
}

func (self templateEngine) genUkko(prefix, frontend, outfile string, database Database) {
  // get the last 15 bumped threads globally
  var threads []ThreadModel
  for _, article := range database.GetLastBumpedThreads("", 15) {
    newsgroup, msgid := article[0], article[1]
    // obtain board
    board := self.obtainBoard(prefix, frontend, newsgroup, database)
    board = board.Update(0, database)
    for _, th := range(board[0].Threads()) {
      if th.OP().MessageID() == msgid {
        threads = append(threads, th)
        break
      }
    }
    // save state of board
    self.groups[newsgroup] = board
  }
  wr, err := OpenFileWriter(outfile)
  if err == nil {
    io.WriteString(wr, template.renderTemplate("ukko.mustache", map[string]interface{} { "prefix" : prefix, "threads" : threads }))
    wr.Close()
    log.Println("wrote file", outfile)
  } else {
    log.Println("error generating ukko", err)
  }
}

func (self templateEngine) genThread(messageID, prefix, frontend, outfile string, db Database) {

  newsgroup, page, err := db.GetPageForRootMessage(messageID)
  if err != nil {
    log.Println(err)
  }
  // get it
  board := self.obtainBoard(prefix, frontend, newsgroup, db)
  // update our thread
  board[page] = board[page].UpdateThread(messageID, db)
  // save it
  self.groups[newsgroup] = board
  for _, th := range board[page].Threads() {
    if th.OP().MessageID() == messageID {
      // we found it
      wr, err := OpenFileWriter(outfile)
      if err == nil {
        th.RenderTo(wr)
        wr.Close()
        log.Println("wrote file", outfile)
      } else {
        log.Println(err)
      }
    }
  }
}

func newTemplateEngine(dir string) *templateEngine {
  return &templateEngine{
    groups: make(map[string]GroupModel),
    templates: make(map[string]string),
    template_dir: dir,
  }
}

var template = newTemplateEngine(defaultTemplateDir())

func renderPostForm(prefix, board, op_msg_id string) string {
  url := prefix + "post/" + board
  button := "New Thread"
  if op_msg_id != "" {
    button = "Reply"
  }
  return template.renderTemplate("postform.mustache", map[string]string { "post_url" : url, "reference" : op_msg_id , "button" : button } )
}

func (self templateEngine) genFrontPage(top_count int, frontend_name, outfile string, db Database) {
  // the graph for the front page
  var frontpage_graph boardPageRows

  // for each group
  groups := db.GetAllNewsgroups()
  for idx, group := range groups {
    if idx >= top_count {
      break
    }
    // posts per hour
    hour := db.CountPostsInGroup(group, 3600)
    // posts per day
    day := db.CountPostsInGroup(group, 86400)
    // posts total
    all := db.CountPostsInGroup(group, 0)
    frontpage_graph = append(frontpage_graph, boardPageRow{
      All: all,
      Day: day,
      Hour: hour,
      Board: group,
    })
  }
  wr, err := OpenFileWriter(outfile)
  if err != nil {
    log.Println("cannot render front page", err)
    return
  }

  param := make(map[string]interface{})
  sort.Sort(frontpage_graph)
  param["graph"] = frontpage_graph
  param["frontend"] = frontend_name
  param["totalposts"] = db.ArticleCount()
  _, err = io.WriteString(wr, self.renderTemplate("frontpage.mustache", param))
  if err != nil {
    log.Println("error writing front page", err)
  }
  wr.Close()
  log.Println("wrote file", outfile)
}
<|endoftext|>