<commit_before>// hooks provides types and constants that define the hooks known to Juju.
package hooks

// Kind enumerates the different kinds of hooks that exist.
type Kind string

const (
	// None of these hooks are ever associated with a relation; each of them
	// represents a change to the state of the unit as a whole. The values
	// themselves are all valid hook names.
	Install       Kind = "install"
	Start         Kind = "start"
	ConfigChanged Kind = "config-changed"
	UpgradeCharm  Kind = "upgrade-charm"
	Stop          Kind = "stop"

	// These hooks require an associated relation, and the name of the relation
	// unit whose change triggered the hook. The hook file names that these
	// kinds represent will be prefixed by the relation name; for example,
	// "db-relation-joined".
	RelationJoined   Kind = "relation-joined"
	RelationChanged  Kind = "relation-changed"
	RelationDeparted Kind = "relation-departed"

	// This hook requires an associated relation. The represented hook file name
	// will be prefixed by the relation name, just like the other Relation* Kind
	// values.
	RelationBroken Kind = "relation-broken"
)

var unitHooks = []Kind{
	Install,
	Start,
	ConfigChanged,
	UpgradeCharm,
	Stop,
}

// UnitHooks returns all known unit hook kinds.
func UnitHooks() []Kind {
	return unitHooks
}

var relationHooks = []Kind{
	RelationJoined,
	RelationChanged,
	RelationDeparted,
	RelationBroken,
}

// RelationHooks returns all known relation hook kinds.
func RelationHooks() []Kind {
	return relationHooks
}

// IsRelation returns whether the Kind represents a relation hook.
func (kind Kind) IsRelation() bool {
	switch kind {
	case RelationJoined, RelationChanged, RelationDeparted, RelationBroken:
		return true
	}
	return false
}
<commit_msg>Made copies of hooks slices before returning<commit_after>// hooks provides types and constants that define the hooks known to Juju.
package hooks

// Kind enumerates the different kinds of hooks that exist.
type Kind string

const (
	// None of these hooks are ever associated with a relation; each of them
	// represents a change to the state of the unit as a whole. The values
	// themselves are all valid hook names.
	Install       Kind = "install"
	Start         Kind = "start"
	ConfigChanged Kind = "config-changed"
	UpgradeCharm  Kind = "upgrade-charm"
	Stop          Kind = "stop"

	// These hooks require an associated relation, and the name of the relation
	// unit whose change triggered the hook. The hook file names that these
	// kinds represent will be prefixed by the relation name; for example,
	// "db-relation-joined".
	RelationJoined   Kind = "relation-joined"
	RelationChanged  Kind = "relation-changed"
	RelationDeparted Kind = "relation-departed"

	// This hook requires an associated relation. The represented hook file name
	// will be prefixed by the relation name, just like the other Relation* Kind
	// values.
	RelationBroken Kind = "relation-broken"
)

var unitHooks = []Kind{
	Install,
	Start,
	ConfigChanged,
	UpgradeCharm,
	Stop,
}

// UnitHooks returns all known unit hook kinds.
func UnitHooks() []Kind {
	hooks := make([]Kind, len(unitHooks))
	copy(hooks, unitHooks)
	return hooks
}

var relationHooks = []Kind{
	RelationJoined,
	RelationChanged,
	RelationDeparted,
	RelationBroken,
}

// RelationHooks returns all known relation hook kinds.
func RelationHooks() []Kind {
	hooks := make([]Kind, len(relationHooks))
	copy(hooks, relationHooks)
	return hooks
}

// IsRelation returns whether the Kind represents a relation hook.
func (kind Kind) IsRelation() bool {
	switch kind {
	case RelationJoined, RelationChanged, RelationDeparted, RelationBroken:
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package logfanout

import (
	"errors"
	"io"
	"sync"

	"github.com/winston-ci/winston/db"
)

type LogFanout struct {
	job   string
	build int
	db    db.DB

	lock *sync.Mutex

	sinks []io.WriteCloser

	closed        bool
	waitForClosed chan struct{}
}

func NewLogFanout(job string, build int, db db.DB) *LogFanout {
	return &LogFanout{
		job:   job,
		build: build,
		db:    db,

		lock:          new(sync.Mutex),
		waitForClosed: make(chan struct{}),
	}
}

func (fanout *LogFanout) Write(data []byte) (int, error) {
	fanout.lock.Lock()

	err := fanout.db.AppendBuildLog(fanout.job, fanout.build, data)
	if err != nil {
		return 0, err
	}

	newSinks := []io.WriteCloser{}
	for _, sink := range fanout.sinks {
		_, err := sink.Write(data)
		if err != nil {
			continue
		}

		newSinks = append(newSinks, sink)
	}

	fanout.sinks = newSinks

	fanout.lock.Unlock()

	return len(data), nil
}

func (fanout *LogFanout) Attach(sink io.WriteCloser) error {
	fanout.lock.Lock()

	log, err := fanout.db.BuildLog(fanout.job, fanout.build)
	if err != nil {
		return err
	}

	_, err = sink.Write(log)
	if err != nil {
		return err
	}

	if fanout.closed {
		sink.Close()
	} else {
		fanout.sinks = append(fanout.sinks, sink)
	}

	fanout.lock.Unlock()

	<-fanout.waitForClosed

	return nil
}

func (fanout *LogFanout) Close() error {
	fanout.lock.Lock()
	defer fanout.lock.Unlock()

	if fanout.closed {
		return errors.New("close twice")
	}

	for _, sink := range fanout.sinks {
		sink.Close()
	}

	fanout.closed = true
	fanout.sinks = nil

	close(fanout.waitForClosed)

	return nil
}
<commit_msg>fix deadlocking test<commit_after>package logfanout

import (
	"errors"
	"io"
	"sync"

	"github.com/winston-ci/winston/db"
)

type LogFanout struct {
	job   string
	build int
	db    db.DB

	lock *sync.Mutex

	sinks []io.WriteCloser

	closed        bool
	waitForClosed chan struct{}
}

func NewLogFanout(job string, build int, db db.DB) *LogFanout {
	return &LogFanout{
		job:   job,
		build: build,
		db:    db,

		lock:          new(sync.Mutex),
		waitForClosed: make(chan struct{}),
	}
}

func (fanout *LogFanout) Write(data []byte) (int, error) {
	fanout.lock.Lock()

	err := fanout.db.AppendBuildLog(fanout.job, fanout.build, data)
	if err != nil {
		return 0, err
	}

	newSinks := []io.WriteCloser{}
	for _, sink := range fanout.sinks {
		_, err := sink.Write(data)
		if err != nil {
			continue
		}

		newSinks = append(newSinks, sink)
	}

	fanout.sinks = newSinks

	fanout.lock.Unlock()

	return len(data), nil
}

func (fanout *LogFanout) Attach(sink io.WriteCloser) error {
	fanout.lock.Lock()

	log, err := fanout.db.BuildLog(fanout.job, fanout.build)
	if err == nil {
		_, err = sink.Write(log)
		if err != nil {
			fanout.lock.Unlock()
			return err
		}
	}

	if fanout.closed {
		sink.Close()
	} else {
		fanout.sinks = append(fanout.sinks, sink)
	}

	fanout.lock.Unlock()

	<-fanout.waitForClosed

	return nil
}

func (fanout *LogFanout) Close() error {
	fanout.lock.Lock()
	defer fanout.lock.Unlock()

	if fanout.closed {
		return errors.New("close twice")
	}

	for _, sink := range fanout.sinks {
		sink.Close()
	}

	fanout.closed = true
	fanout.sinks = nil

	close(fanout.waitForClosed)

	return nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/gzip"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idSize    = 8 // should be between 6 and 256
	siteUrl   = "http://localhost:9090"
	listen    = "localhost:9090"
	indexTmpl = "index.html"
	dataDir   = "data"
	maxSize   = 1 << 20 // before compression
	minLife   = 1 * time.Minute
	defLife   = 1 * time.Hour
	maxLife   = 12 * time.Hour

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
	invalidLife = "The lifetime specified is invalid (units: s,m,h)."
	smallLife   = "The lifetime specified is too small (min: %s)."
	largeLife   = "The lifetime specified is too large (max: %s)."
)

const chars = "abcdefghijklmnopqrstuvwxyz0123456789"

var validId *regexp.Regexp = regexp.MustCompile("^[a-z0-9]{" + strconv.FormatInt(idSize, 10) + "}$")

var indexTemplate *template.Template

func pathId(id string) string {
	return path.Join(id[0:2], id[2:4], id[4:])
}

func randomId() string {
	s := make([]byte, idSize)
	var offset uint = 0
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				return string(s)
			}
			r >>= 8
		}
	}
	return strings.Repeat(chars[0:1], idSize)
}

func endLife(id string) {
	pastePath := pathId(id)
	err := os.Remove(pastePath)
	if err == nil {
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		timer := time.NewTimer(minLife)
		go func() {
			<-timer.C
			endLife(id)
		}()
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		id := r.URL.Path[1:]
		if len(id) == 0 {
			indexTemplate.Execute(w, siteUrl)
			return
		}
		if !validId.MatchString(id) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		pastePath := pathId(id)
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		compReader, err := gzip.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		io.Copy(w, compReader)
		compReader.Close()
		pasteFile.Close()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, maxSize)
		var id, pastePath string
		for {
			id = randomId()
			pastePath = pathId(id)
			if _, err := os.Stat(pastePath); os.IsNotExist(err) {
				break
			}
		}
		if err = r.ParseMultipartForm(maxSize << 1); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		var life time.Duration
		var content string
		if vs, found := r.Form["life"]; !found {
			life = defLife
		} else {
			life, err = time.ParseDuration(vs[0])
			if err != nil {
				w.WriteHeader(http.StatusBadRequest)
				fmt.Fprintf(w, "%s\n", invalidLife)
				return
			}
		}
		if life < minLife || life > maxLife {
			w.WriteHeader(http.StatusBadRequest)
			if life < minLife {
				fmt.Fprintf(w, "%s\n", smallLife, minLife)
			} else {
				fmt.Fprintf(w, "%s\n", largeLife, maxLife)
			}
			return
		}
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		timer := time.NewTimer(life)
		go func() {
			<-timer.C
			endLife(id)
		}()
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		compWriter := gzip.NewWriter(pasteFile)
		_, err = io.WriteString(compWriter, content)
		compWriter.Close()
		pasteFile.Close()
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		log.Printf("Created a new paste: %s (lifetime: %s)", id, life)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func main() {
	var err error
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Printf("Could not load template %s: %s", indexTmpl, err)
		return
	}
	if err = os.RemoveAll(dataDir); err != nil {
		log.Printf("Could not clean data directory %s: %s", dataDir, err)
		return
	}
	if err = os.Mkdir(dataDir, 0700); err != nil {
		log.Printf("Could not create data directory %s: %s", dataDir, err)
		return
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Printf("Could not enter data directory %s: %s", dataDir, err)
		return
	}
	http.HandleFunc("/", handler)
	http.ListenAndServe(listen, nil)
}
<commit_msg>Replace gzip with the lighter zlib<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idSize    = 8 // should be between 6 and 256
	siteUrl   = "http://localhost:9090"
	listen    = "localhost:9090"
	indexTmpl = "index.html"
	dataDir   = "data"
	maxSize   = 1 << 20 // before compression
	minLife   = 1 * time.Minute
	defLife   = 1 * time.Hour
	maxLife   = 12 * time.Hour

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
	invalidLife = "The lifetime specified is invalid (units: s,m,h)."
	smallLife   = "The lifetime specified is too small (min: %s)."
	largeLife   = "The lifetime specified is too large (max: %s)."
)

const chars = "abcdefghijklmnopqrstuvwxyz0123456789"

var validId *regexp.Regexp = regexp.MustCompile("^[a-z0-9]{" + strconv.FormatInt(idSize, 10) + "}$")

var indexTemplate *template.Template

func pathId(id string) string {
	return path.Join(id[0:2], id[2:4], id[4:])
}

func randomId() string {
	s := make([]byte, idSize)
	var offset uint = 0
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				return string(s)
			}
			r >>= 8
		}
	}
	return strings.Repeat(chars[0:1], idSize)
}

func endLife(id string) {
	pastePath := pathId(id)
	err := os.Remove(pastePath)
	if err == nil {
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		timer := time.NewTimer(minLife)
		go func() {
			<-timer.C
			endLife(id)
		}()
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		id := r.URL.Path[1:]
		if len(id) == 0 {
			indexTemplate.Execute(w, siteUrl)
			return
		}
		if !validId.MatchString(id) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		pastePath := pathId(id)
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		io.Copy(w, compReader)
		compReader.Close()
		pasteFile.Close()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, maxSize)
		var id, pastePath string
		for {
			id = randomId()
			pastePath = pathId(id)
			if _, err := os.Stat(pastePath); os.IsNotExist(err) {
				break
			}
		}
		if err = r.ParseMultipartForm(maxSize << 1); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		var life time.Duration
		var content string
		if vs, found := r.Form["life"]; !found {
			life = defLife
		} else {
			life, err = time.ParseDuration(vs[0])
			if err != nil {
				w.WriteHeader(http.StatusBadRequest)
				fmt.Fprintf(w, "%s\n", invalidLife)
				return
			}
		}
		if life < minLife || life > maxLife {
			w.WriteHeader(http.StatusBadRequest)
			if life < minLife {
				fmt.Fprintf(w, "%s\n", smallLife, minLife)
			} else {
				fmt.Fprintf(w, "%s\n", largeLife, maxLife)
			}
			return
		}
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		timer := time.NewTimer(life)
		go func() {
			<-timer.C
			endLife(id)
		}()
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		compWriter := zlib.NewWriter(pasteFile)
		_, err = io.WriteString(compWriter, content)
		compWriter.Close()
		pasteFile.Close()
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		log.Printf("Created a new paste: %s (lifetime: %s)", id, life)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func main() {
	var err error
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Printf("Could not load template %s: %s", indexTmpl, err)
		return
	}
	if err = os.RemoveAll(dataDir); err != nil {
		log.Printf("Could not clean data directory %s: %s", dataDir, err)
		return
	}
	if err = os.Mkdir(dataDir, 0700); err != nil {
		log.Printf("Could not create data directory %s: %s", dataDir, err)
		return
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Printf("Could not enter data directory %s: %s", dataDir, err)
		return
	}
	http.HandleFunc("/", handler)
	http.ListenAndServe(listen, nil)
}
<|endoftext|>


<commit_before>package dnsmasq

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/version"

	"golang.org/x/sys/unix"
)

// DHCPAllocation represents an IP allocation from dnsmasq.
type DHCPAllocation struct {
	IP     net.IP
	Name   string
	MAC    net.HardwareAddr
	Static bool
}

// ConfigMutex used to coordinate access to the dnsmasq config files.
var ConfigMutex sync.Mutex

// UpdateStaticEntry writes a single dhcp-host line for a network/instance combination.
func UpdateStaticEntry(network string, projectName string, instanceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error {
	line := hwaddr

	// Generate the dhcp-host line
	if ipv4Address != "" {
		line += fmt.Sprintf(",%s", ipv4Address)
	}

	if ipv6Address != "" {
		line += fmt.Sprintf(",[%s]", ipv6Address)
	}

	if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" {
		line += fmt.Sprintf(",%s", instanceName)
	}

	if line == hwaddr {
		return nil
	}

	err := ioutil.WriteFile(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)), []byte(line+"\n"), 0644)
	if err != nil {
		return err
	}

	return nil
}

// RemoveStaticEntry removes a single dhcp-host line for a network/instance combination.
func RemoveStaticEntry(network string, projectName string, instanceName string) error {
	err := os.Remove(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)))
	if err != nil {
		return err
	}

	return nil
}

// Kill kills dnsmasq for a particular network (or optionally reloads it).
func Kill(name string, reload bool) error {
	// Check if we have a running dnsmasq at all
	pidPath := shared.VarPath("networks", name, "dnsmasq.pid")
	if !shared.PathExists(pidPath) {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Grab the PID
	content, err := ioutil.ReadFile(pidPath)
	if err != nil {
		return err
	}
	pid := strings.TrimSpace(string(content))

	// Check for empty string
	if pid == "" {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if the process still exists
	if !shared.PathExists(fmt.Sprintf("/proc/%s", pid)) {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if it's dnsmasq
	cmdPath, err := os.Readlink(fmt.Sprintf("/proc/%s/exe", pid))
	if err != nil {
		cmdPath = ""
	}

	// Deal with deleted paths
	cmdName := filepath.Base(strings.Split(cmdPath, " ")[0])
	if cmdName != "dnsmasq" {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		os.Remove(pidPath)
		return nil
	}

	// Parse the pid
	pidInt, err := strconv.Atoi(pid)
	if err != nil {
		return err
	}

	// Actually kill the process
	if reload {
		err = unix.Kill(pidInt, unix.SIGHUP)
		if err != nil {
			return err
		}

		return nil
	}

	err = unix.Kill(pidInt, unix.SIGKILL)
	if err != nil {
		return err
	}

	// Cleanup
	os.Remove(pidPath)
	return nil
}

// GetVersion returns the version of dnsmasq.
func GetVersion() (*version.DottedVersion, error) {
	// Discard stderr on purpose (occasional linker errors)
	output, err := exec.Command("dnsmasq", "--version").Output()
	if err != nil {
		return nil, fmt.Errorf("Failed to check dnsmasq version: %v", err)
	}

	lines := strings.Split(string(output), " ")
	return version.NewDottedVersion(lines[2])
}

// DHCPStaticIPs retrieves the dnsmasq statically allocated IPs for a container.
// Returns IPv4 and IPv6 DHCPAllocation structs respectively.
func DHCPStaticIPs(network string, containerName string) (DHCPAllocation, DHCPAllocation, error) {
	var IPv4, IPv6 DHCPAllocation

	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.hosts") + "/" + containerName)
	if err != nil {
		return IPv4, IPv6, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.SplitN(scanner.Text(), ",", -1)
		for _, field := range fields {
			// Check if field is IPv4 or IPv6 address.
			if strings.Count(field, ".") == 3 {
				IP := net.ParseIP(field)
				if IP.To4() == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv4 = DHCPAllocation{Name: containerName, Static: true, IP: IP.To4()}

			} else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") {
				IP := net.ParseIP(field[1 : len(field)-1])
				if IP == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv6 = DHCPAllocation{Name: containerName, Static: true, IP: IP}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4, IPv6, err
	}

	return IPv4, IPv6, nil
}

// DHCPAllocatedIPs returns a map of IPs currently allocated (statically and dynamically)
// in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing
// the net.IP format. The value of each map item is a DHCPAllocation struct containing at least
// whether the allocation was static or dynamic and optionally container name or MAC address.
// MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable).
// Static allocations are not overridden by dynamic allocations, allowing for container name to be
// included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be
// reliably linked to containers using either name or MAC because dnsmasq does not record the MAC
// address for these records, and the recorded host name can be set by the container if the dns.mode
// for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return
// any identifying info.
func DHCPAllocatedIPs(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) {
	IPv4s := make(map[[4]byte]DHCPAllocation)
	IPv6s := make(map[[16]byte]DHCPAllocation)

	// First read all statically allocated IPs.
	files, err := ioutil.ReadDir(shared.VarPath("networks", network, "dnsmasq.hosts"))
	if err != nil {
		return IPv4s, IPv6s, err
	}

	for _, entry := range files {
		IPv4, IPv6, err := DHCPStaticIPs(network, entry.Name())
		if err != nil {
			return IPv4s, IPv6s, err
		}

		if IPv4.IP != nil {
			var IPKey [4]byte
			copy(IPKey[:], IPv4.IP.To4())
			IPv4s[IPKey] = IPv4
		}

		if IPv6.IP != nil {
			var IPKey [16]byte
			copy(IPKey[:], IPv6.IP.To16())
			IPv6s[IPKey] = IPv6
		}
	}

	// Next read all dynamic allocated IPs.
	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.leases"))
	if err != nil {
		return IPv4s, IPv6s, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 5 {
			IP := net.ParseIP(fields[2])
			if IP == nil {
				return IPv4s, IPv6s, fmt.Errorf("Error parsing IP address: %v", fields[2])
			}

			// Handle IPv6 addresses.
			if IP.To4() == nil {
				var IPKey [16]byte
				copy(IPKey[:], IP.To16())

				// Don't replace IPs from static config as more reliable.
				if IPv6s[IPKey].Name != "" {
					continue
				}

				IPv6s[IPKey] = DHCPAllocation{
					Static: false,
					IP:     IP.To16(),
				}
			} else {
				// MAC only available in IPv4 leases.
				MAC, err := net.ParseMAC(fields[1])
				if err != nil {
					return IPv4s, IPv6s, err
				}

				var IPKey [4]byte
				copy(IPKey[:], IP.To4())

				// Don't replace IPs from static config as more reliable.
				if IPv4s[IPKey].Name != "" {
					continue
				}

				IPv4s[IPKey] = DHCPAllocation{
					MAC:    MAC,
					Static: false,
					IP:     IP.To4(),
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4s, IPv6s, err
	}

	return IPv4s, IPv6s, nil
}
<commit_msg>lxd/dnsmasq: Don't fail file deletion if missing<commit_after>package dnsmasq

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/version"

	"golang.org/x/sys/unix"
)

// DHCPAllocation represents an IP allocation from dnsmasq.
type DHCPAllocation struct {
	IP     net.IP
	Name   string
	MAC    net.HardwareAddr
	Static bool
}

// ConfigMutex used to coordinate access to the dnsmasq config files.
var ConfigMutex sync.Mutex

// UpdateStaticEntry writes a single dhcp-host line for a network/instance combination.
func UpdateStaticEntry(network string, projectName string, instanceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error {
	line := hwaddr

	// Generate the dhcp-host line
	if ipv4Address != "" {
		line += fmt.Sprintf(",%s", ipv4Address)
	}

	if ipv6Address != "" {
		line += fmt.Sprintf(",[%s]", ipv6Address)
	}

	if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" {
		line += fmt.Sprintf(",%s", instanceName)
	}

	if line == hwaddr {
		return nil
	}

	err := ioutil.WriteFile(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)), []byte(line+"\n"), 0644)
	if err != nil {
		return err
	}

	return nil
}

// RemoveStaticEntry removes a single dhcp-host line for a network/instance combination.
func RemoveStaticEntry(network string, projectName string, instanceName string) error {
	err := os.Remove(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)))
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Kill kills dnsmasq for a particular network (or optionally reloads it).
func Kill(name string, reload bool) error {
	// Check if we have a running dnsmasq at all
	pidPath := shared.VarPath("networks", name, "dnsmasq.pid")
	if !shared.PathExists(pidPath) {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Grab the PID
	content, err := ioutil.ReadFile(pidPath)
	if err != nil {
		return err
	}
	pid := strings.TrimSpace(string(content))

	// Check for empty string
	if pid == "" {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if the process still exists
	if !shared.PathExists(fmt.Sprintf("/proc/%s", pid)) {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if it's dnsmasq
	cmdPath, err := os.Readlink(fmt.Sprintf("/proc/%s/exe", pid))
	if err != nil {
		cmdPath = ""
	}

	// Deal with deleted paths
	cmdName := filepath.Base(strings.Split(cmdPath, " ")[0])
	if cmdName != "dnsmasq" {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		os.Remove(pidPath)
		return nil
	}

	// Parse the pid
	pidInt, err := strconv.Atoi(pid)
	if err != nil {
		return err
	}

	// Actually kill the process
	if reload {
		err = unix.Kill(pidInt, unix.SIGHUP)
		if err != nil {
			return err
		}

		return nil
	}

	err = unix.Kill(pidInt, unix.SIGKILL)
	if err != nil {
		return err
	}

	// Cleanup
	os.Remove(pidPath)
	return nil
}

// GetVersion returns the version of dnsmasq.
func GetVersion() (*version.DottedVersion, error) {
	// Discard stderr on purpose (occasional linker errors)
	output, err := exec.Command("dnsmasq", "--version").Output()
	if err != nil {
		return nil, fmt.Errorf("Failed to check dnsmasq version: %v", err)
	}

	lines := strings.Split(string(output), " ")
	return version.NewDottedVersion(lines[2])
}

// DHCPStaticIPs retrieves the dnsmasq statically allocated IPs for a container.
// Returns IPv4 and IPv6 DHCPAllocation structs respectively.
func DHCPStaticIPs(network string, containerName string) (DHCPAllocation, DHCPAllocation, error) {
	var IPv4, IPv6 DHCPAllocation

	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.hosts") + "/" + containerName)
	if err != nil {
		return IPv4, IPv6, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.SplitN(scanner.Text(), ",", -1)
		for _, field := range fields {
			// Check if field is IPv4 or IPv6 address.
			if strings.Count(field, ".") == 3 {
				IP := net.ParseIP(field)
				if IP.To4() == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv4 = DHCPAllocation{Name: containerName, Static: true, IP: IP.To4()}

			} else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") {
				IP := net.ParseIP(field[1 : len(field)-1])
				if IP == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv6 = DHCPAllocation{Name: containerName, Static: true, IP: IP}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4, IPv6, err
	}

	return IPv4, IPv6, nil
}

// DHCPAllocatedIPs returns a map of IPs currently allocated (statically and dynamically)
// in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing
// the net.IP format. The value of each map item is a DHCPAllocation struct containing at least
// whether the allocation was static or dynamic and optionally container name or MAC address.
// MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable).
// Static allocations are not overridden by dynamic allocations, allowing for container name to be
// included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be
// reliably linked to containers using either name or MAC because dnsmasq does not record the MAC
// address for these records, and the recorded host name can be set by the container if the dns.mode
// for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return
// any identifying info.
func DHCPAllocatedIPs(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) {
	IPv4s := make(map[[4]byte]DHCPAllocation)
	IPv6s := make(map[[16]byte]DHCPAllocation)

	// First read all statically allocated IPs.
	files, err := ioutil.ReadDir(shared.VarPath("networks", network, "dnsmasq.hosts"))
	if err != nil {
		return IPv4s, IPv6s, err
	}

	for _, entry := range files {
		IPv4, IPv6, err := DHCPStaticIPs(network, entry.Name())
		if err != nil {
			return IPv4s, IPv6s, err
		}

		if IPv4.IP != nil {
			var IPKey [4]byte
			copy(IPKey[:], IPv4.IP.To4())
			IPv4s[IPKey] = IPv4
		}

		if IPv6.IP != nil {
			var IPKey [16]byte
			copy(IPKey[:], IPv6.IP.To16())
			IPv6s[IPKey] = IPv6
		}
	}

	// Next read all dynamic allocated IPs.
	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.leases"))
	if err != nil {
		return IPv4s, IPv6s, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 5 {
			IP := net.ParseIP(fields[2])
			if IP == nil {
				return IPv4s, IPv6s, fmt.Errorf("Error parsing IP address: %v", fields[2])
			}

			// Handle IPv6 addresses.
			if IP.To4() == nil {
				var IPKey [16]byte
				copy(IPKey[:], IP.To16())

				// Don't replace IPs from static config as more reliable.
				if IPv6s[IPKey].Name != "" {
					continue
				}

				IPv6s[IPKey] = DHCPAllocation{
					Static: false,
					IP:     IP.To16(),
				}
			} else {
				// MAC only available in IPv4 leases.
				MAC, err := net.ParseMAC(fields[1])
				if err != nil {
					return IPv4s, IPv6s, err
				}

				var IPKey [4]byte
				copy(IPKey[:], IP.To4())

				// Don't replace IPs from static config as more reliable.
				if IPv4s[IPKey].Name != "" {
					continue
				}

				IPv4s[IPKey] = DHCPAllocation{
					MAC:    MAC,
					Static: false,
					IP:     IP.To4(),
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4s, IPv6s, err
	}

	return IPv4s, IPv6s, nil
}
<|endoftext|>


<commit_before>package hilbert

// Peano represents a 2D Peano curve of order N for mapping to and from.
// Implements SpaceFilling interface.
type Peano struct {
	N int // Always a power of three, and is the width/height of the space.
}

// isPow3 returns true if n is a power of 3.
func isPow3(n float64) bool {
	// I wanted to do the following, but due to subtle floating point issues it didn't work
	// const ln3 = 1.098612288668109691395245236922525704647490557822749451734694333637494 // https://oeis.org/A002391
	//return n == math.Pow(3, math.Trunc(math.Log(n) / ln3))
	for n >= 1 {
		if n == 1 {
			return true
		}
		n = n / 3
	}
	return false
}

// NewPeano returns a new Peano space filling curve which maps integers to and from the curve.
// n must be a power of three.
func NewPeano(n int) (*Peano, error) {
	if n <= 0 {
		return nil, ErrNotPositive
	}

	if !isPow3(float64(n)) {
		return nil, ErrNotPowerOfThree
	}

	return &Peano{
		N: n,
	}, nil
}

// GetDimensions returns the width and height of the 2D space.
func (p *Peano) GetDimensions() (int, int) {
	return p.N, p.N
}

// Map transforms a one dimension value, t, in the range [0, n^3-1] to coordinates on the Peano
// curve in the two-dimension space, where x and y are within [0,n-1].
func (p *Peano) Map(t int) (x, y int, err error) {
	if t < 0 || t >= p.N*p.N {
		return -1, -1, ErrOutOfRange
	}

	for i := 1; i < p.N; i = i * 3 {
		s := t % 9

		// rx/ry are the coordinates in the 3x3 grid
		rx := int(s / 3)
		ry := int(s % 3)
		if rx == 1 {
			ry = 2 - ry
		}

		// now based on depth rotate our points
		if i > 1 {
			x, y = p.rotate(i, x, y, s)
		}

		x += rx * i
		y += ry * i

		t /= 9
	}

	return x, y, nil
}

// rotate rotates the x and y coordinates depending on the current n depth.
func (p *Peano) rotate(n, x, y, s int) (int, int) {

	if n == 1 {
		// Special case
		return x, y
	}

	n = n - 1
	switch s {
	case 0:
		return x, y // normal
	case 1:
		return n - x, y // fliph
	case 2:
		return x, y // normal
	case 3:
		return x, n - y // flipv
	case 4:
		return n - x, n - y // flipv and fliph
	case 5:
		return x, n - y // flipv
	case 6:
		return x, y // normal
	case 7:
		return n - x, y // fliph
	case 8:
		return x, y // normal
	}

	panic("assertion failure: this line should never be reached")
}

// MapInverse transform coordinates on the Peano curve from (x,y) to t.
// NOTE IMPLEMENTED YET
func (p *Peano) MapInverse(x, y int) (t int, err error) {
	if x < 0 || x >= p.N || y < 0 || y >= p.N {
		return -1, ErrOutOfRange
	}

	panic("Not finished")
	return -1, nil
}
<commit_msg>Fixed minor typo<commit_after>package hilbert

// Peano represents a 2D Peano curve of order N for mapping to and from.
// Implements SpaceFilling interface.
type Peano struct {
	N int // Always a power of three, and is the width/height of the space.
}

// isPow3 returns true if n is a power of 3.
func isPow3(n float64) bool {
	// I wanted to do the following, but due to subtle floating point issues it didn't work
	// const ln3 = 1.098612288668109691395245236922525704647490557822749451734694333637494 // https://oeis.org/A002391
	//return n == math.Pow(3, math.Trunc(math.Log(n) / ln3))
	for n >= 1 {
		if n == 1 {
			return true
		}
		n = n / 3
	}
	return false
}

// NewPeano returns a new Peano space filling curve which maps integers to and from the curve.
// n must be a power of three.
func NewPeano(n int) (*Peano, error) {
	if n <= 0 {
		return nil, ErrNotPositive
	}

	if !isPow3(float64(n)) {
		return nil, ErrNotPowerOfThree
	}

	return &Peano{
		N: n,
	}, nil
}

// GetDimensions returns the width and height of the 2D space.
func (p *Peano) GetDimensions() (int, int) {
	return p.N, p.N
}

// Map transforms a one dimension value, t, in the range [0, n^3-1] to coordinates on the Peano
// curve in the two-dimension space, where x and y are within [0,n-1].
func (p *Peano) Map(t int) (x, y int, err error) {
	if t < 0 || t >= p.N*p.N {
		return -1, -1, ErrOutOfRange
	}

	for i := 1; i < p.N; i = i * 3 {
		s := t % 9

		// rx/ry are the coordinates in the 3x3 grid
		rx := int(s / 3)
		ry := int(s % 3)
		if rx == 1 {
			ry = 2 - ry
		}

		// now based on depth rotate our points
		if i > 1 {
			x, y = p.rotate(i, x, y, s)
		}

		x += rx * i
		y += ry * i

		t /= 9
	}

	return x, y, nil
}

// rotate rotates the x and y coordinates depending on the current n depth.
func (p *Peano) rotate(n, x, y, s int) (int, int) {

	if n == 1 {
		// Special case
		return x, y
	}

	n = n - 1
	switch s {
	case 0:
		return x, y // normal
	case 1:
		return n - x, y // fliph
	case 2:
		return x, y // normal
	case 3:
		return x, n - y // flipv
	case 4:
		return n - x, n - y // flipv and fliph
	case 5:
		return x, n - y // flipv
	case 6:
		return x, y // normal
	case 7:
		return n - x, y // fliph
	case 8:
		return x, y // normal
	}

	panic("assertion failure: this line should never be reached")
}

// MapInverse transform coordinates on the Peano curve from (x,y) to t.
// NOT IMPLEMENTED YET
func (p *Peano) MapInverse(x, y int) (t int, err error) {
	if x < 0 || x >= p.N || y < 0 || y >= p.N {
		return -1, ErrOutOfRange
	}

	panic("Not finished")
	return -1, nil
}
<|endoftext|>


<commit_before>package lzb

import (
	"fmt"
	"testing"
)

func TestNewReadBuffer(t *testing.T) {
	r, err := newReadBuffer(10, 10)
	if err != nil {
		t.Fatalf("newReadBuffer error %s", err)
	}
	c := r.capacity()
	if c != 10 {
		t.Errorf("capacity is %d; want %d", c, 10)
	}
	s := r.dict.size
	if s != 10 {
		t.Errorf("dict size is %d; want %d", s, 10)
	}
}

func mustNewReadBuffer(capacity, histsize int64) *readBuffer {
	b, err := newReadBuffer(capacity, histsize)
	if err != nil {
		panic(fmt.Errorf("newReadBuffer error %s", err))
	}
	return b
}

func TestReadBuffer_Seek(t *testing.T) {
	r := mustNewReadBuffer(10, 10)
	p := []byte("abcdef")
	n, err := r.Write(p)
	if err != nil {
		t.Fatalf("r.Write(%q) error %s", p, err)
	}
	if n != len(p) {
		t.Fatalf("r.Write(%q) returned %d; want %d", p, n, len(p))
	}
	tests := []struct {
		offset int64
		whence int
		off    int64
		err    error
	}{
		{2, 0, 2, nil},
		{1, 1, 3, nil},
		{-1, 2, 5, nil},
		{0, 0, 0, nil},
		{-1, 0, 0, errOffset},
		{6, 0, 6, nil},
		{7, 0, 6, errOffset},
	}
	for _, c := range tests {
		off, err := r.Seek(c.offset, c.whence)
		if err != c.err {
			t.Errorf("r.Seek(%d, %d) returned error %s; want %s",
				c.offset, c.whence, err, c.err)
		}
		if off != c.off {
			t.Errorf("r.Seek(%d, %d) returned offset %d; want %d",
				c.offset, c.whence, off, c.off)
		}
	}
}
<commit_msg>lzb: added test case for test of readBuffer.Seek<commit_after>package lzb

import (
	"fmt"
	"testing"
)

func TestNewReadBuffer(t *testing.T) {
	r, err := newReadBuffer(10, 10)
	if err != nil {
		t.Fatalf("newReadBuffer error %s", err)
	}
	c := r.capacity()
	if c != 10 {
		t.Errorf("capacity is %d; want %d", c, 10)
	}
	s := r.dict.size
	if s != 10 {
		t.Errorf("dict size is %d; want %d", s, 10)
	}
}

func mustNewReadBuffer(capacity, histsize int64) *readBuffer {
	b, err := newReadBuffer(capacity, histsize)
	if err != nil {
		panic(fmt.Errorf("newReadBuffer error %s", err))
	}
	return b
}

func TestReadBuffer_Seek(t *testing.T) {
	r := mustNewReadBuffer(10, 10)
	p := []byte("abcdef")
	n, err := r.Write(p)
	if err != nil {
		t.Fatalf("r.Write(%q) error %s", p, err)
	}
	if n != len(p) {
		t.Fatalf("r.Write(%q) returned %d; want %d", p, n, len(p))
	}
	tests := []struct {
		offset int64
		whence int
		off    int64
		err    error
	}{
		{2, 0, 2, nil},
		{1, 1, 3, nil},
		{-1, 2, 5, nil},
		{0, 0, 0, nil},
		{-1, 0, 0, errOffset},
		{6, 0, 6, nil},
		{7, 0, 6, errOffset},
		{5, 3, 6, errWhence},
	}
	for _, c := range tests {
		off, err := r.Seek(c.offset, c.whence)
		if err != c.err {
			t.Errorf("r.Seek(%d, %d) returned error %s; want %s",
				c.offset, c.whence, err, c.err)
		}
		if off != c.off {
			t.Errorf("r.Seek(%d, %d) returned offset %d; want %d",
				c.offset, c.whence, off, c.off)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"

	. "github.com/russross/codegrinder/types"
)

func init() {
	problemTypes["standardmlunittest"] = &ProblemType{
		Name:        "standardmlunittest",
		Image:       "codegrinder/standardml",
		MaxCPU:      10,
		MaxSession:  30 * 60,
		MaxTimeout:  5 * 60,
		MaxFD:       100,
		MaxFileSize: 10,
		MaxMemory:   128,
		MaxThreads:  20,
		Actions: map[string]*ProblemTypeAction{
			"grade": &ProblemTypeAction{
				Action:      "grade",
				Button:      "Grade",
				Message:     "Grading‥",
				Interactive: false,
				Handler:     nannyHandler(standardMLUnittestGrade),
			},
			"run": &ProblemTypeAction{
				Action:      "run",
				Button:      "Run",
				Message:     "Running %s‥",
				Interactive: true,
				Handler:     nannyHandler(standardMLRun),
			},
			"shell": &ProblemTypeAction{
				Action:      "shell",
				Button:      "Shell",
				Message:     "Running PolyML shell‥",
				Interactive: true,
				Handler:     nannyHandler(standardMLShell),
			},
		},
	}
}

var standardMLaout = `#!/bin/bash
set -e
ln -s tests/*.sml ./
rm -f test_detail.xml
poly < tests.sml
`

func standardMLUnittestGrade(n *Nanny, args, options []string, files map[string]string, stdin io.Reader) {
	log.Printf("standard ML unit test grade")

	// create an a.out file
	if err := n.PutFiles(map[string]string{"a.out": standardMLaout}); err != nil {
		n.ReportCard.LogAndFailf("error creating a.out: %v", err)
		return
	}

	// make it executable
	if err := n.ExecSimple([]string{"chmod", "755", "a.out"}, nil, false); err != nil {
		return
	}

	// run a.out and parse the output (in common with c++)
	gTestAOutCommon(n, files, nil)
}

var standardMLRunaout = `#!/bin/bash
set -e
echo ';' > /tmp/semi
cat *.sml /tmp/semi | rlwrap poly
`

func standardMLRun(n *Nanny, args, options []string, files map[string]string, stdin io.Reader) {
	log.Printf("standard ML run")

	// create an a.out file
	if err := n.PutFiles(map[string]string{"a.out": standardMLaout}); err != nil {
		n.ReportCard.LogAndFailf("error creating a.out: %v", err)
		return
	}

	// make it executable
	if err := n.ExecSimple([]string{"chmod", "755", "a.out"}, nil, false); err != nil {
		return
	}

	// run a.out
	n.ExecSimple([]string{"a.out"}, stdin, true)
}

func standardMLShell(n *Nanny, args, options []string, files map[string]string, stdin io.Reader) {
	log.Printf("standard ML shell")

	n.ExecSimple([]string{"rlwrap", "poly"}, stdin, true)
}
<commit_msg>sml work<commit_after>package main

import (
	"io"
	"log"

	. "github.com/russross/codegrinder/types"
)

func init() {
	problemTypes["standardmlunittest"] = &ProblemType{
		Name:        "standardmlunittest",
		Image:       "codegrinder/standardml",
		MaxCPU:      10,
		MaxSession:  30 * 60,
		MaxTimeout:  5 * 60,
		MaxFD:       100,
		MaxFileSize: 10,
		MaxMemory:   128,
		MaxThreads:  20,
		Actions: map[string]*ProblemTypeAction{
			"grade": &ProblemTypeAction{
				Action:      "grade",
				Button:      "Grade",
				Message:     "Grading‥",
				Interactive: false,
				Handler:     nannyHandler(standardMLUnittestGrade),
			},
			"run": &ProblemTypeAction{
				Action:      "run",
				Button:      "Run",
				Message:     "Running %s‥",
				Interactive: true,
				Handler:     nannyHandler(standardMLRun),
			},
			"shell": &ProblemTypeAction{
				Action:      "shell",
				Button:      "Shell",
				Message:     "Running PolyML shell‥",
				Interactive: true,
				Handler:     nannyHandler(standardMLShell),
			},
		},
	}
}

var standardMLaout = `#!/bin/bash
set -e
ln -s tests/*.sml ./
rm -f test_detail.xml
poly < tests.sml
`

func standardMLUnittestGrade(n *Nanny, args, options []string, files map[string]string, stdin io.Reader) {
	log.Printf("standard ML unit test grade")

	// create an a.out file
	if err := n.PutFiles(map[string]string{"a.out": standardMLaout}); err != nil {
		n.ReportCard.LogAndFailf("error creating a.out: %v", err)
		return
	}

	// make it executable
	if err := n.ExecSimple([]string{"chmod", "755", "a.out"}, nil, false); err != nil {
		return
	}

	// run a.out and parse the output (in common with c++)
	gTestAOutCommon(n, files, nil)
}

var standardMLRunaout = `#!/bin/bash
set -e
echo ';' > /tmp/semi
cat *.sml /tmp/semi | poly
`

func standardMLRun(n *Nanny, args, options []string, files map[string]string, stdin io.Reader) {
	log.Printf("standard ML run")

	// create an a.out file
	if err := n.PutFiles(map[string]string{"a.out": standardMLaout}); err != nil {
		n.ReportCard.LogAndFailf("error creating a.out: %v", err)
		return
	}

	// make it executable
	if err := n.ExecSimple([]string{"chmod", "755", "a.out"}, nil, false); err != nil {
		return
	}

	// run a.out
	n.ExecSimple([]string{"./a.out"}, stdin, true)
}

func standardMLShell(n *Nanny, args, options []string, files map[string]string, stdin io.Reader) {
	log.Printf("standard ML shell")

	n.ExecSimple([]string{"poly"}, stdin, true)
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/otto/appfile"
	"github.com/hashicorp/otto/appfile/detect"
	"github.com/hashicorp/otto/ui"
)

// CompileCommand is the command that is responsible for "compiling" the
// Appfile into a set of data that is used by the other commands for
// execution.
type CompileCommand struct {
	Meta

	Detectors []*detect.Detector
}

func (c *CompileCommand) Run(args []string) int {
	var flagAppfile string
	fs := c.FlagSet("compile", FlagSetNone)
	fs.Usage = func() { c.Ui.Error(c.Help()) }
	fs.StringVar(&flagAppfile, "appfile", "", "")
	if err := fs.Parse(args); err != nil {
		return 1
	}

	// Load a UI
	ui := c.OttoUi()
	ui.Header("Loading Appfile...")

	// Determine all the Appfile paths
	//
	// First, if an Appfile was specified on the command-line, it must
	// exist so we validate that it exists.
	if flagAppfile != "" {
		fi, err := os.Stat(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}

		if fi.IsDir() {
			flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
		}
	}

	// If the Appfile is still blank, just use our current directory
	if flagAppfile == "" {
		var err error
		flagAppfile, err = os.Getwd()
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading working directory: %s", err))
			return 1
		}

		flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
	}

	// If we have the Appfile, then make sure it is an absoute path
	if flagAppfile != "" {
		var err error
		flagAppfile, err = filepath.Abs(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error getting Appfile path: %s", err))
			return 1
		}
	}

	// Load the appfile. This is the only time we ever load the
	// raw Appfile. All other commands load the compiled Appfile.
	var app *appfile.File
	if fi, err := os.Stat(flagAppfile); err == nil && !fi.IsDir() {
		app, err = appfile.ParseFile(flagAppfile)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
	}

	// Tell the user what is happening if they have no Appfile
	if app == nil {
		ui.Header("No Appfile found! Detecting project information...")
		ui.Message(fmt.Sprintf(
			"No Appfile was found. If there is no Appfile, Otto will do its best\n" +
				"to detect the type of application this is and set reasonable defaults.\n" +
				"This is a good way to get started with Otto, but over time we recommend\n" +
				"writing a real Appfile since this will allow more complex customizations,\n" +
				"the ability to reference dependencies, versioning, and more."))
	}

	// Load the default Appfile so we can merge in any defaults into
	// the loaded Appfile (if there is one).
	detectConfig := &detect.Config{
		Detectors: c.Detectors,
	}
	appDef, err := appfile.Default(filepath.Dir(flagAppfile), detectConfig)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading Appfile: %s", err))
		return 1
	}

	// If there was no loaded Appfile and we don't have an application
	// type then we weren't able to detect the type. Error.
	if app == nil && appDef.Application.Type == "" {
		c.Ui.Error(strings.TrimSpace(errCantDetectType))
		return 1
	}

	// Merge the appfiles
	if app != nil {
		if err := appDef.Merge(app); err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}
	}
	app = appDef

	// Compile the Appfile
	ui.Header("Fetching all Appfile dependencies...")
	capp, err := appfile.Compile(app, &appfile.CompileOpts{
		Dir: filepath.Join(
			filepath.Dir(app.Path), DefaultOutputDir, DefaultOutputDirCompiledAppfile),
		Callback: c.compileCallback(ui),
	})
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling Appfile: %s", err))
		return 1
	}

	// Get a core
	core, err := c.Core(capp)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading core: %s", err))
		return 1
	}

	// Get the active infrastructure just for UI reasons
	infra := app.ActiveInfrastructure()

	// Before the compilation, output to the user what is going on
	ui.Header("Compiling...")
	ui.Message(fmt.Sprintf(
		"Application:    %s (%s)",
		app.Application.Name,
		app.Application.Type))
	ui.Message(fmt.Sprintf("Project:        %s", app.Project.Name))
	ui.Message(fmt.Sprintf(
		"Infrastructure: %s (%s)",
		infra.Type,
		infra.Flavor))
	ui.Message("")

	// Compile!
	if err := core.Compile(); err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling: %s", err))
		return 1
	}

	// Success!
	ui.Header("Compilation success!")
	ui.Message(fmt.Sprintf(
		"This means that Otto is now ready to start a development environment,\n" +
			"deploy this application, build the supporting infastructure, and\n" +
			"more. See the help for more information.\n\n" +
			"Supporting files to enable Otto to manage your application from\n" +
			"development to deployment have been placed in the output directory.\n" +
			"These files can be manually inspected to determine what Otto will do."))

	return 0
}

func (c *CompileCommand) Synopsis() string {
	return "Prepares your project for being run."
}

func (c *CompileCommand) Help() string {
	helpText := `
Usage: otto [options] [path]

  Compiles the Appfile into the set of supporting files used for
  development, deploy, etc. If path is not specified, the current directory
  is assumed.

`

	return strings.TrimSpace(helpText)
}

func (c *CompileCommand) compileCallback(ui ui.Ui) func(appfile.CompileEvent) {
	return func(raw appfile.CompileEvent) {
		switch e := raw.(type) {
		case *appfile.CompileEventDep:
			ui.Message(fmt.Sprintf(
				"Fetching dependency: %s", e.Source))
		}
	}
}

const errCantDetectType = `
No Appfile is present and Otto couldn't detect the project type automatically.
Otto does its best without an Appfile to detect what kind of project this is
automatically, but sometimes this fails if the project is in a structure
Otto doesn't recognize or its a project type that Otto doesn't yet support.

Please create an Appfile and specify at a minimum the project type. Below
is an example minimal Appfile specifying the "go" project type:

    application {
        type = "go"
    }

If you believe Otto should've been able to automatically detect your
project type, then please open an issue with the Otto project.
`
<commit_msg>command/compile: output import status<commit_after>package command

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/otto/appfile"
	"github.com/hashicorp/otto/appfile/detect"
	"github.com/hashicorp/otto/ui"
)

// CompileCommand is the command that is responsible for "compiling" the
// Appfile into a set of data that is used by the other commands for
// execution.
type CompileCommand struct {
	Meta

	Detectors []*detect.Detector
}

func (c *CompileCommand) Run(args []string) int {
	var flagAppfile string
	fs := c.FlagSet("compile", FlagSetNone)
	fs.Usage = func() { c.Ui.Error(c.Help()) }
	fs.StringVar(&flagAppfile, "appfile", "", "")
	if err := fs.Parse(args); err != nil {
		return 1
	}

	// Load a UI
	ui := c.OttoUi()
	ui.Header("Loading Appfile...")

	// Determine all the Appfile paths
	//
	// First, if an Appfile was specified on the command-line, it must
	// exist so we validate that it exists.
	if flagAppfile != "" {
		fi, err := os.Stat(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}

		if fi.IsDir() {
			flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
		}
	}

	// If the Appfile is still blank, just use our current directory
	if flagAppfile == "" {
		var err error
		flagAppfile, err = os.Getwd()
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading working directory: %s", err))
			return 1
		}

		flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
	}

	// If we have the Appfile, then make sure it is an absoute path
	if flagAppfile != "" {
		var err error
		flagAppfile, err = filepath.Abs(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error getting Appfile path: %s", err))
			return 1
		}
	}

	// Load the appfile. This is the only time we ever load the
	// raw Appfile. All other commands load the compiled Appfile.
	var app *appfile.File
	if fi, err := os.Stat(flagAppfile); err == nil && !fi.IsDir() {
		app, err = appfile.ParseFile(flagAppfile)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
	}

	// Tell the user what is happening if they have no Appfile
	if app == nil {
		ui.Header("No Appfile found! Detecting project information...")
		ui.Message(fmt.Sprintf(
			"No Appfile was found. If there is no Appfile, Otto will do its best\n" +
				"to detect the type of application this is and set reasonable defaults.\n" +
				"This is a good way to get started with Otto, but over time we recommend\n" +
				"writing a real Appfile since this will allow more complex customizations,\n" +
				"the ability to reference dependencies, versioning, and more."))
	}

	// Load the default Appfile so we can merge in any defaults into
	// the loaded Appfile (if there is one).
	detectConfig := &detect.Config{
		Detectors: c.Detectors,
	}
	appDef, err := appfile.Default(filepath.Dir(flagAppfile), detectConfig)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading Appfile: %s", err))
		return 1
	}

	// If there was no loaded Appfile and we don't have an application
	// type then we weren't able to detect the type. Error.
	if app == nil && appDef.Application.Type == "" {
		c.Ui.Error(strings.TrimSpace(errCantDetectType))
		return 1
	}

	// Merge the appfiles
	if app != nil {
		if err := appDef.Merge(app); err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}
	}
	app = appDef

	// Compile the Appfile
	ui.Header("Fetching all Appfile dependencies...")
	capp, err := appfile.Compile(app, &appfile.CompileOpts{
		Dir: filepath.Join(
			filepath.Dir(app.Path), DefaultOutputDir, DefaultOutputDirCompiledAppfile),
		Callback: c.compileCallback(ui),
	})
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling Appfile: %s", err))
		return 1
	}

	// Get a core
	core, err := c.Core(capp)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading core: %s", err))
		return 1
	}

	// Get the active infrastructure just for UI reasons
	infra := app.ActiveInfrastructure()

	// Before the compilation, output to the user what is going on
	ui.Header("Compiling...")
	ui.Message(fmt.Sprintf(
		"Application:    %s (%s)",
		app.Application.Name,
		app.Application.Type))
	ui.Message(fmt.Sprintf("Project:        %s", app.Project.Name))
	ui.Message(fmt.Sprintf(
		"Infrastructure: %s (%s)",
		infra.Type,
		infra.Flavor))
	ui.Message("")

	// Compile!
	if err := core.Compile(); err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling: %s", err))
		return 1
	}

	// Success!
	ui.Header("Compilation success!")
	ui.Message(fmt.Sprintf(
		"This means that Otto is now ready to start a development environment,\n" +
			"deploy this application, build the supporting infastructure, and\n" +
			"more. See the help for more information.\n\n" +
			"Supporting files to enable Otto to manage your application from\n" +
			"development to deployment have been placed in the output directory.\n" +
			"These files can be manually inspected to determine what Otto will do."))

	return 0
}

func (c *CompileCommand) Synopsis() string {
	return "Prepares your project for being run."
}

func (c *CompileCommand) Help() string {
	helpText := `
Usage: otto [options] [path]

  Compiles the Appfile into the set of supporting files used for
  development, deploy, etc. If path is not specified, the current directory
  is assumed.

`

	return strings.TrimSpace(helpText)
}

func (c *CompileCommand) compileCallback(ui ui.Ui) func(appfile.CompileEvent) {
	return func(raw appfile.CompileEvent) {
		switch e := raw.(type) {
		case *appfile.CompileEventDep:
			ui.Message(fmt.Sprintf(
				"Fetching dependency: %s", e.Source))
		case *appfile.CompileEventImport:
			ui.Message(fmt.Sprintf(
				"Fetching import: %s", e.Source))
		}
	}
}

const errCantDetectType = `
No Appfile is present and Otto couldn't detect the project type automatically.
Otto does its best without an Appfile to detect what kind of project this is
automatically, but sometimes this fails if the project is in a structure
Otto doesn't recognize or its a project type that Otto doesn't yet support.

Please create an Appfile and specify at a minimum the project type. Below
is an example minimal Appfile specifying the "go" project type:

    application {
        type = "go"
    }

If you believe Otto should've been able to automatically detect your
project type, then please open an issue with the Otto project.
`
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/naoty/contrib/util"
)

func Contrib(c *cli.Context) {
	if len(c.Args()) == 0 {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	names, err := getContributors(c.Args())
	if err != nil {
		log.Fatal(err)
	}

	number := c.Int("number")
	if number == 0 {
		number = len(names)
	}

	for i, name := range names {
		if i >= number {
			break
		}
		fmt.Println(name)
	}

	os.Exit(0)
}

var ContribFlag = cli.IntFlag{
	Name:  "number, n",
	Usage: "the number of contributors",
}

func getContributors(filenames []string) ([]string, error) {
	var err error
	contributors := make(map[string]int)

	for _, filename := range filenames {
		names, err := gitLog(filename)
		if err != nil {
			continue
		}

		for _, name := range names {
			contributors[name] += 1
		}
	}

	if err != nil {
		return []string{}, err
	}

	vs := util.NewValSorter(contributors)
	vs.Sort()

	return vs.Keys, nil
}

func gitLog(filename string) ([]string, error) {
	// Print only auther's name
	cmd := exec.Command("git", "log", "--pretty=format:%an", filename)
	out, err := cmd.CombinedOutput()
	str := string(out)
	str = strings.Trim(str, "\n")
	return strings.Split(str, "\n"), err
}
<commit_msg>Set default number of contributors to 1<commit_after>package command

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/naoty/contrib/util"
)

func Contrib(c *cli.Context) {
	if len(c.Args()) == 0 {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	names, err := getContributors(c.Args())
	if err != nil {
		log.Fatal(err)
	}

	number := c.Int("number")
	if number == 0 {
		number = 1
	}

	for i, name := range names {
		if i >= number {
			break
		}
		fmt.Println(name)
	}

	os.Exit(0)
}

var ContribFlag = cli.IntFlag{
	Name:  "number, n",
	Usage: "the number of contributors",
}

func getContributors(filenames []string) ([]string, error) {
	var err error
	contributors := make(map[string]int)

	for _, filename := range filenames {
		names, err := gitLog(filename)
		if err != nil {
			continue
		}

		for _, name := range names {
			contributors[name] += 1
		}
	}

	if err != nil {
		return []string{}, err
	}

	vs := util.NewValSorter(contributors)
	vs.Sort()

	return vs.Keys, nil
}

func gitLog(filename string) ([]string, error) {
	// Print only auther's name
	cmd := exec.Command("git", "log", "--pretty=format:%an", filename)
	out, err := cmd.CombinedOutput()
	str := string(out)
	str = strings.Trim(str, "\n")
	return strings.Split(str, "\n"), err
}
<|endoftext|>


<commit_before>package main

import (
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/jackc/pgx"

	"fknsrs.biz/p/jishaku-web/lib/scraper"
)

var (
	SELECT_QUERY = `select "info_hash", "trackers" from "torrents" where "torrents"."last_scrape" is null or "torrents"."last_scrape" < (now() - interval '1 day') order by "torrents"."last_scrape" asc nulls first, "torrents"."first_seen" desc limit 1000`
	INSERT_QUERY = `insert into "scrapes" ("info_hash", "tracker", "time", "success", "downloaded", "complete", "incomplete") values ($1, $2, now(), $3, $4, $5, $6)`
	UPDATE_QUERY = `update "torrents" set "last_scrape" = now() where "info_hash" = $1`
)

func scraperCommandFunction(databaseDSN string, debug bool) {
	connConfig, err := pgx.ParseDSN(databaseDSN)
	if err != nil {
		panic(err)
	}

	dbConfig := pgx.ConnPoolConfig{
		ConnConfig:     connConfig,
		MaxConnections: 4,
	}

	if debug {
		dbConfig.Logger = (*wrappedLogger)(logrus.StandardLogger())
	}

	db, err := pgx.NewConnPool(dbConfig)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	s := scraper.New()
	go s.Run()

	for {
		rows, err := db.Query(SELECT_QUERY)
		if err != nil {
			panic(err)
		}

		var wg1 sync.WaitGroup

		for rows.Next() {
			var infoHash string
			var trackers []string
			if err := rows.Scan(&infoHash, &trackers); err != nil {
				panic(err)
				// continue
			}

			wg1.Add(1)

			go func(infoHash string, trackers []string) {
				defer wg1.Done()

				var wg2 sync.WaitGroup

				h, err := scraper.HashFromString(infoHash)
				if err != nil {
					panic(err)
				}

				for _, t := range trackers {
					wg2.Add(1)

					go func(t string) {
						defer wg2.Done()

						l := logrus.WithFields(logrus.Fields{
							"info_hash": infoHash,
							"tracker":   t,
						})

						res, err := s.Scrape(t, h)
						if err != nil {
							if _, ok := err.(scraper.UnsupportedProtocolError); !ok {
								l.Error(err.Error())
							}

							if _, err := db.Exec(INSERT_QUERY, infoHash, t, false, 0, 0, 0); err != nil {
								panic(err)
							}

							return
						}

						if _, err := db.Exec(INSERT_QUERY, infoHash, t, true, res.Downloaded, res.Complete, res.Incomplete); err != nil {
							panic(err)
						}
					}(t)
				}

				wg2.Wait()

				if _, err := db.Exec(UPDATE_QUERY, infoHash); err != nil {
					panic(err)
				}
			}(infoHash, trackers)
		}

		logrus.Info("waiting")

		wg1.Wait()
	}
}
<commit_msg>pause if there's no work to do<commit_after>package main

import (
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/jackc/pgx"

	"fknsrs.biz/p/jishaku-web/lib/scraper"
)

var (
	SELECT_QUERY = `select "info_hash", "trackers" from "torrents" where "torrents"."last_scrape" is null or "torrents"."last_scrape" < (now() - interval '1 day') order by "torrents"."last_scrape" asc nulls first, "torrents"."first_seen" desc limit 1000`
	INSERT_QUERY = `insert into "scrapes" ("info_hash", "tracker", "time", "success", "downloaded", "complete", "incomplete") values ($1, $2, now(), $3, $4, $5, $6)`
	UPDATE_QUERY = `update "torrents" set "last_scrape" = now() where "info_hash" = $1`
)

func scraperCommandFunction(databaseDSN string, debug bool) {
	connConfig, err := pgx.ParseDSN(databaseDSN)
	if err != nil {
		panic(err)
	}

	dbConfig := pgx.ConnPoolConfig{
		ConnConfig:     connConfig,
		MaxConnections: 4,
	}

	if debug {
		dbConfig.Logger = (*wrappedLogger)(logrus.StandardLogger())
	}

	db, err := pgx.NewConnPool(dbConfig)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	s := scraper.New()
	go s.Run()

	for {
		rows, err := db.Query(SELECT_QUERY)
		if err != nil {
			panic(err)
		}

		var wg1 sync.WaitGroup

		var c int
		for rows.Next() {
			c++

			var infoHash string
			var trackers []string
			if err := rows.Scan(&infoHash, &trackers); err != nil {
				panic(err)
				// continue
			}

			wg1.Add(1)

			go func(infoHash string, trackers []string) {
				defer wg1.Done()

				var wg2 sync.WaitGroup

				h, err := scraper.HashFromString(infoHash)
				if err != nil {
					panic(err)
				}

				for _, t := range trackers {
					wg2.Add(1)

					go func(t string) {
						defer wg2.Done()

						l := logrus.WithFields(logrus.Fields{
							"info_hash": infoHash,
							"tracker":   t,
						})

						res, err := s.Scrape(t, h)
						if err != nil {
							if _, ok := err.(scraper.UnsupportedProtocolError); !ok {
								l.Error(err.Error())
							}

							if _, err := db.Exec(INSERT_QUERY, infoHash, t, false, 0, 0, 0); err != nil {
								panic(err)
							}

							return
						}

						if _, err := db.Exec(INSERT_QUERY, infoHash, t, true, res.Downloaded, res.Complete, res.Incomplete); err != nil {
							panic(err)
						}
					}(t)
				}

				wg2.Wait()

				if _, err := db.Exec(UPDATE_QUERY, infoHash); err != nil {
					panic(err)
				}
			}(infoHash, trackers)
		}

		logrus.WithField("count", c).Info("waiting")

		wg1.Wait()

		if c == 0 {
			time.Sleep(time.Minute)
		}
	}
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/anacrolix/utp"
	"github.com/gophergala2016/meshbird/network/protocol"
	"log"
	"net"
)

type ListenerService struct {
	BaseService

	localNode *LocalNode
	socket    *utp.Socket
}

func (l ListenerService) Name() string {
	return "listener"
}

func (l *ListenerService) Init(ln *LocalNode) error {
	log.Printf("Listening on port: %d", ln.State().ListenPort+1)
	socket, err := utp.NewSocket("udp4", fmt.Sprintf("0.0.0.0:%d", ln.State().ListenPort+1))
	if err != nil {
		return err
	}

	l.localNode = ln
	l.socket = socket
	return nil
}

func (l *ListenerService) Run() error {
	for {
		conn, err := l.socket.Accept()
		if err != nil {
			break
		}

		log.Printf("Has new connection: %s", conn.RemoteAddr().String())

		if err = l.process(conn); err != nil {
			log.Printf("Error on process: %s", err)
		}
	}
	return nil
}

func (l *ListenerService) Stop() {
	l.SetStatus(StatusStopping)
	l.socket.Close()
}

func (l *ListenerService) process(c net.Conn) error {
	defer c.Close()

	handshakeMsg, errHandshake := protocol.ReadDecodeHandshake(c)
	if errHandshake != nil {
		return errHandshake
	}

	log.Println("Processing hansdhake...")

	if !protocol.IsMagicValid(handshakeMsg.Bytes()) {
		return fmt.Errorf("Invalid magic bytes")
	}

	log.Println("Magic bytes are correct. Preparing reply...")

	if err := protocol.WriteEncodeOk(c); err != nil {
		return err
	}
	if err := protocol.WriteEncodePeerInfo(c, l.localNode.State().PrivateIP); err != nil {
		return err
	}

	peerInfo, errPeerInfo := protocol.ReadDecodePeerInfo(c)
	if errPeerInfo != nil {
		return errPeerInfo
	}

	log.Println("Processing PeerInfo...")

	rn := NewRemoteNode(c, handshakeMsg.SessionKey(), peerInfo.PrivateIP())

	netTable, ok := l.localNode.Service("net-table").(*NetTable)
	if !ok || netTable == nil {
		return fmt.Errorf("net-table is nil")
	}

	netTable.AddRemoteNode(rn)

	return nil
}
<commit_msg>Commented defer s.Close()<commit_after>package common

import (
	"fmt"
	"github.com/anacrolix/utp"
	"github.com/gophergala2016/meshbird/network/protocol"
	"log"
	"net"
)

type ListenerService struct {
	BaseService

	localNode *LocalNode
	socket    *utp.Socket
}

func (l ListenerService) Name() string {
	return "listener"
}

func (l *ListenerService) Init(ln *LocalNode) error {
	log.Printf("Listening on port: %d", ln.State().ListenPort+1)
	socket, err := utp.NewSocket("udp4", fmt.Sprintf("0.0.0.0:%d", ln.State().ListenPort+1))
	if err != nil {
		return err
	}

	l.localNode = ln
	l.socket = socket
	return nil
}

func (l *ListenerService) Run() error {
	for {
		conn, err := l.socket.Accept()
		if err != nil {
			break
		}

		log.Printf("Has new connection: %s", conn.RemoteAddr().String())

		if err = l.process(conn); err != nil {
			log.Printf("Error on process: %s", err)
		}
	}
	return nil
}

func (l *ListenerService) Stop() {
	l.SetStatus(StatusStopping)
	l.socket.Close()
}

func (l *ListenerService) process(c net.Conn) error {
	//defer c.Close()

	handshakeMsg, errHandshake := protocol.ReadDecodeHandshake(c)
	if errHandshake != nil {
		return errHandshake
	}

	log.Println("Processing hansdhake...")

	if !protocol.IsMagicValid(handshakeMsg.Bytes()) {
		return fmt.Errorf("Invalid magic bytes")
	}

	log.Println("Magic bytes are correct. Preparing reply...")

	if err := protocol.WriteEncodeOk(c); err != nil {
		return err
	}
	if err := protocol.WriteEncodePeerInfo(c, l.localNode.State().PrivateIP); err != nil {
		return err
	}

	peerInfo, errPeerInfo := protocol.ReadDecodePeerInfo(c)
	if errPeerInfo != nil {
		return errPeerInfo
	}

	log.Println("Processing PeerInfo...")

	rn := NewRemoteNode(c, handshakeMsg.SessionKey(), peerInfo.PrivateIP())

	netTable, ok := l.localNode.Service("net-table").(*NetTable)
	if !ok || netTable == nil {
		return fmt.Errorf("net-table is nil")
	}

	netTable.AddRemoteNode(rn)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package expr provides nftables rule expressions.
package expr

import (
	"github.com/google/nftables/binaryutil"
	"github.com/mdlayher/netlink"
	"golang.org/x/sys/unix"
)

// Marshal serializes the specified expression into a byte slice.
func Marshal(e Any) ([]byte, error) {
	return e.marshal()
}

// Any is an interface implemented by any expression type.
type Any interface {
	marshal() ([]byte, error)
}

// MetaKey specifies which piece of meta information should be loaded. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Matching_packet_metainformation
type MetaKey uint32

// Possible MetaKey values.
const (
	MetaKeyLEN        MetaKey = unix.NFT_META_LEN
	MetaKeyPROTOCOL   MetaKey = unix.NFT_META_PROTOCOL
	MetaKeyPRIORITY   MetaKey = unix.NFT_META_PRIORITY
	MetaKeyMARK       MetaKey = unix.NFT_META_MARK
	MetaKeyIIF        MetaKey = unix.NFT_META_IIF
	MetaKeyOIF        MetaKey = unix.NFT_META_OIF
	MetaKeyIIFNAME    MetaKey = unix.NFT_META_IIFNAME
	MetaKeyOIFNAME    MetaKey = unix.NFT_META_OIFNAME
	MetaKeyIIFTYPE    MetaKey = unix.NFT_META_IIFTYPE
	MetaKeyOIFTYPE    MetaKey = unix.NFT_META_OIFTYPE
	MetaKeySKUID      MetaKey = unix.NFT_META_SKUID
	MetaKeySKGID      MetaKey = unix.NFT_META_SKGID
	MetaKeyNFTRACE    MetaKey = unix.NFT_META_NFTRACE
	MetaKeyRTCLASSID  MetaKey = unix.NFT_META_RTCLASSID
	MetaKeySECMARK    MetaKey = unix.NFT_META_SECMARK
	MetaKeyNFPROTO    MetaKey = unix.NFT_META_NFPROTO
	MetaKeyL4PROTO    MetaKey = unix.NFT_META_L4PROTO
	MetaKeyBRIIIFNAME MetaKey = unix.NFT_META_BRI_IIFNAME
	MetaKeyBRIOIFNAME MetaKey = unix.NFT_META_BRI_OIFNAME
	MetaKeyPKTTYPE    MetaKey = unix.NFT_META_PKTTYPE
	MetaKeyCPU        MetaKey = unix.NFT_META_CPU
	MetaKeyIIFGROUP   MetaKey = unix.NFT_META_IIFGROUP
	MetaKeyOIFGROUP   MetaKey = unix.NFT_META_OIFGROUP
	MetaKeyCGROUP     MetaKey = unix.NFT_META_CGROUP
	MetaKeyPRANDOM    MetaKey = unix.NFT_META_PRANDOM
)

// Meta loads packet meta information for later comparisons. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Matching_packet_metainformation
type Meta struct {
	Key      MetaKey
	Register uint32
}

func (e *Meta) marshal() ([]byte, error) {
	exprData, err := netlink.MarshalAttributes(
		[]netlink.Attribute{
			{Type: unix.NFTA_META_KEY, Data: binaryutil.BigEndian.PutUint32(uint32(e.Key))},
			{Type: unix.NFTA_META_DREG, Data: binaryutil.BigEndian.PutUint32(e.Register)},
		},
	)
	if err != nil {
		return nil, err
	}
	return netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_EXPR_NAME, Data: []byte("meta\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: exprData},
	})
}

// Masq (Masquerade) is a special case of SNAT, where the source address is
// automagically set to the address of the output interface. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Performing_Network_Address_Translation_(NAT)#Masquerading
type Masq struct{}

func (e *Masq) marshal() ([]byte, error) {
	return netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_EXPR_NAME, Data: []byte("masq\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: nil},
	})
}

// CmpOp specifies which type of comparison should be performed.
type CmpOp uint32

// Possible CmpOp values.
const (
	CmpOpEq  CmpOp = unix.NFT_CMP_EQ
	CmpOpNeq CmpOp = unix.NFT_CMP_NEQ
	CmpOpLt  CmpOp = unix.NFT_CMP_LT
	CmpOpLte CmpOp = unix.NFT_CMP_LTE
	CmpOpGt  CmpOp = unix.NFT_CMP_GT
	CmpOpGte CmpOp = unix.NFT_CMP_GTE
)

// Cmp compares a register with the specified data.
type Cmp struct {
	Op       CmpOp
	Register uint32
	Data     []byte
}

func (e *Cmp) marshal() ([]byte, error) {
	cmpData, err := netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_DATA_VALUE, Data: e.Data},
	})
	if err != nil {
		return nil, err
	}
	exprData, err := netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_CMP_SREG, Data: binaryutil.BigEndian.PutUint32(e.Register)},
		{Type: unix.NFTA_CMP_OP, Data: binaryutil.BigEndian.PutUint32(unix.NFT_CMP_EQ)},
		{Type: unix.NLA_F_NESTED | unix.NFTA_CMP_DATA, Data: cmpData},
	})
	if err != nil {
		return nil, err
	}
	return netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_EXPR_NAME, Data: []byte("cmp\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: exprData},
	})
}
<commit_msg>Cmp: fix accidentally hard-coded EQ operator<commit_after>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package expr provides nftables rule expressions.
package expr

import (
	"github.com/google/nftables/binaryutil"
	"github.com/mdlayher/netlink"
	"golang.org/x/sys/unix"
)

// Marshal serializes the specified expression into a byte slice.
func Marshal(e Any) ([]byte, error) {
	return e.marshal()
}

// Any is an interface implemented by any expression type.
type Any interface {
	marshal() ([]byte, error)
}

// MetaKey specifies which piece of meta information should be loaded. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Matching_packet_metainformation
type MetaKey uint32

// Possible MetaKey values.
const (
	MetaKeyLEN        MetaKey = unix.NFT_META_LEN
	MetaKeyPROTOCOL   MetaKey = unix.NFT_META_PROTOCOL
	MetaKeyPRIORITY   MetaKey = unix.NFT_META_PRIORITY
	MetaKeyMARK       MetaKey = unix.NFT_META_MARK
	MetaKeyIIF        MetaKey = unix.NFT_META_IIF
	MetaKeyOIF        MetaKey = unix.NFT_META_OIF
	MetaKeyIIFNAME    MetaKey = unix.NFT_META_IIFNAME
	MetaKeyOIFNAME    MetaKey = unix.NFT_META_OIFNAME
	MetaKeyIIFTYPE    MetaKey = unix.NFT_META_IIFTYPE
	MetaKeyOIFTYPE    MetaKey = unix.NFT_META_OIFTYPE
	MetaKeySKUID      MetaKey = unix.NFT_META_SKUID
	MetaKeySKGID      MetaKey = unix.NFT_META_SKGID
	MetaKeyNFTRACE    MetaKey = unix.NFT_META_NFTRACE
	MetaKeyRTCLASSID  MetaKey = unix.NFT_META_RTCLASSID
	MetaKeySECMARK    MetaKey = unix.NFT_META_SECMARK
	MetaKeyNFPROTO    MetaKey = unix.NFT_META_NFPROTO
	MetaKeyL4PROTO    MetaKey = unix.NFT_META_L4PROTO
	MetaKeyBRIIIFNAME MetaKey = unix.NFT_META_BRI_IIFNAME
	MetaKeyBRIOIFNAME MetaKey = unix.NFT_META_BRI_OIFNAME
	MetaKeyPKTTYPE    MetaKey = unix.NFT_META_PKTTYPE
	MetaKeyCPU        MetaKey = unix.NFT_META_CPU
	MetaKeyIIFGROUP   MetaKey = unix.NFT_META_IIFGROUP
	MetaKeyOIFGROUP   MetaKey = unix.NFT_META_OIFGROUP
	MetaKeyCGROUP     MetaKey = unix.NFT_META_CGROUP
	MetaKeyPRANDOM    MetaKey = unix.NFT_META_PRANDOM
)

// Meta loads packet meta information for later comparisons. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Matching_packet_metainformation
type Meta struct {
	Key      MetaKey
	Register uint32
}

func (e *Meta) marshal() ([]byte, error) {
	exprData, err := netlink.MarshalAttributes(
		[]netlink.Attribute{
			{Type: unix.NFTA_META_KEY, Data: binaryutil.BigEndian.PutUint32(uint32(e.Key))},
			{Type: unix.NFTA_META_DREG, Data: binaryutil.BigEndian.PutUint32(e.Register)},
		},
	)
	if err != nil {
		return nil, err
	}
	return netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_EXPR_NAME, Data: []byte("meta\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: exprData},
	})
}

// Masq (Masquerade) is a special case of SNAT, where the source address is
// automagically set to the address of the output interface. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Performing_Network_Address_Translation_(NAT)#Masquerading
type Masq struct{}

func (e *Masq) marshal() ([]byte, error) {
	return netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_EXPR_NAME, Data: []byte("masq\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: nil},
	})
}

// CmpOp specifies which type of comparison should be performed.
type CmpOp uint32

// Possible CmpOp values.
const (
	CmpOpEq  CmpOp = unix.NFT_CMP_EQ
	CmpOpNeq CmpOp = unix.NFT_CMP_NEQ
	CmpOpLt  CmpOp = unix.NFT_CMP_LT
	CmpOpLte CmpOp = unix.NFT_CMP_LTE
	CmpOpGt  CmpOp = unix.NFT_CMP_GT
	CmpOpGte CmpOp = unix.NFT_CMP_GTE
)

// Cmp compares a register with the specified data.
type Cmp struct {
	Op       CmpOp
	Register uint32
	Data     []byte
}

func (e *Cmp) marshal() ([]byte, error) {
	cmpData, err := netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_DATA_VALUE, Data: e.Data},
	})
	if err != nil {
		return nil, err
	}
	exprData, err := netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_CMP_SREG, Data: binaryutil.BigEndian.PutUint32(e.Register)},
		{Type: unix.NFTA_CMP_OP, Data: binaryutil.BigEndian.PutUint32(uint32(e.Op))},
		{Type: unix.NLA_F_NESTED | unix.NFTA_CMP_DATA, Data: cmpData},
	})
	if err != nil {
		return nil, err
	}
	return netlink.MarshalAttributes([]netlink.Attribute{
		{Type: unix.NFTA_EXPR_NAME, Data: []byte("cmp\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: exprData},
	})
}
<|endoftext|>


<commit_before>package colly

import (
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// NewHTMLElementFromSelectionNode creates a HTMLElement from a goquery.Selection Node.
func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement {
	return &HTMLElement{
		Name:       n.Data,
		Request:    resp.Request,
		Response:   resp,
		Text:       goquery.NewDocumentFromNode(n).Text(),
		DOM:        s,
		attributes: n.Attr,
	}
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// ChildText returns the concatenated and stripped text content of the matching
// elements.
func (h *HTMLElement) ChildText(goquerySelector string) string {
	return strings.TrimSpace(h.DOM.Find(goquerySelector).Text())
}

// ChildAttr returns the stripped text content of the first matching
// element's attribute.
func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string {
	if attr, ok := h.DOM.Find(goquerySelector).Attr(attrName); ok {
		return strings.TrimSpace(attr)
	}
	return ""
}

// ChildAttrs returns the stripped text content of all the matching
// element's attributes.
func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string {
	res := make([]string, 0)
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		if attr, ok := s.Attr(attrName); ok {
			res = append(res, strings.TrimSpace(attr))
		}
	})
	return res
}

// ForEach iterates over the elements matched by the first argument
// and calls the callback function on every HTMLElement match.
func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement)) {
	i := 0
	h.DOM.Find("table.basic-info-table tr").Each(func(_ int, s *goquery.Selection) {
		for _, n := range s.Nodes {
			callback(i, NewHTMLElementFromSelectionNode(h.Response, s, n))
			i++
		}
	})
}
<commit_msg>There was some cruft from the example code here :)<commit_after>package colly

import (
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// NewHTMLElementFromSelectionNode creates a HTMLElement from a goquery.Selection Node.
func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement {
	return &HTMLElement{
		Name:       n.Data,
		Request:    resp.Request,
		Response:   resp,
		Text:       goquery.NewDocumentFromNode(n).Text(),
		DOM:        s,
		attributes: n.Attr,
	}
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// ChildText returns the concatenated and stripped text content of the matching
// elements.
func (h *HTMLElement) ChildText(goquerySelector string) string {
	return strings.TrimSpace(h.DOM.Find(goquerySelector).Text())
}

// ChildAttr returns the stripped text content of the first matching
// element's attribute.
func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string {
	if attr, ok := h.DOM.Find(goquerySelector).Attr(attrName); ok {
		return strings.TrimSpace(attr)
	}
	return ""
}

// ChildAttrs returns the stripped text content of all the matching
// element's attributes.
func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string {
	res := make([]string, 0)
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		if attr, ok := s.Attr(attrName); ok {
			res = append(res, strings.TrimSpace(attr))
		}
	})
	return res
}

// ForEach iterates over the elements matched by the first argument
// and calls the callback function on every HTMLElement match.
func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement)) {
	i := 0
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		for _, n := range s.Nodes {
			callback(i, NewHTMLElementFromSelectionNode(h.Response, s, n))
			i++
		}
	})
}
<|endoftext|>


<commit_before>// Package hyperloglog implements the HyperLogLog algorithm for
// cardinality estimation. In English: it counts things. It counts
// things using very small amounts of memory compared to the number of
// objects it is counting.
//
// For a full description of the algorithm, see the paper HyperLogLog:
// the analysis of a near-optimal cardinality estimation algorithm by
// Flajolet, et. al.
package hyperloglog

import (
	"fmt"
	"math"
)

type HyperLogLog struct {
	m         uint    // Number of registers
	b         uint32  // Number of bits used to determine register index
	alpha     float64 // Bias correction constant
	registers []uint8
}

// Compute bias correction alpha_m.
func get_alpha(m uint) (result float64) {
	switch m {
	case 16:
		result = 0.673
	case 32:
		result = 0.697
	case 64:
		result = 0.709
	default:
		result = 0.7213 / (1.0 + 1.079/float64(m))
	}
	return result
}

// Return a new HyperLogLog with the given number of registers. More
// registers leads to lower error in your estimated count, at the
// expense of memory.
//
// Choose a power of two number of registers, depending on the amount
// of memory you're willing to use and the error you're willing to
// tolerate. Each register uses one byte of memory.
//
// Approximate error will be:
//     1.04 / sqrt(registers)
//
func New(registers uint) (*HyperLogLog, error) {
	if registers%2 != 0 {
		return nil, fmt.Errorf("number of registers %d not a power of two", registers)
	}
	h := &HyperLogLog{}
	h.m = registers
	h.b = uint32(math.Ceil(math.Log2(float64(registers))))
	h.alpha = get_alpha(registers)
	h.Reset()
	return h, nil
}

// Reset all internal variables and set the count to zero.
func (h *HyperLogLog) Reset() {
	h.registers = make([]uint8, h.m)
}

// Calculate the position of the rightmost 1-bit.
func rho(val uint32, max uint32) uint8 {
	r := 1
	for val&1 == 0 && val <= max {
		r++
		val >>= 1
	}
	return uint8(r)
}

// Add to the count. val should be a 32 bit integer from a good hash
// function.
func (h *HyperLogLog) Add(val uint32) {
	k := 32 - h.b
	r := rho(val, k)
	j := val >> uint(k)
	if r > h.registers[j] {
		h.registers[j] = r
	}
}

// Get the estimated count.
func (h *HyperLogLog) Count() uint64 {
	sum := 0.0
	for _, val := range h.registers {
		sum += 1 / math.Pow(2, float64(val))
	}
	estimate := h.alpha * float64(h.m*h.m) / sum
	pow_2_32 := math.Pow(2, 32)
	if estimate <= 5/2*float64(h.m) {
		// Small range correction
		v := 0
		for _, r := range h.registers {
			if r == 0 {
				v++
			}
		}
		if v > 0 {
			estimate = float64(h.m) * math.Log(float64(h.m)/float64(v))
		}
	} else if estimate > 1/30*pow_2_32 {
		// Large range correction
		estimate = -pow_2_32 * math.Log(1-estimate/pow_2_32)
	}
	return uint64(estimate)
}
<commit_msg>Fix accidental integer divison in range correction conditions<commit_after>// Package hyperloglog implements the HyperLogLog algorithm for
// cardinality estimation. In English: it counts things. It counts
// things using very small amounts of memory compared to the number of
// objects it is counting.
//
// For a full description of the algorithm, see the paper HyperLogLog:
// the analysis of a near-optimal cardinality estimation algorithm by
// Flajolet, et. al.
package hyperloglog

import (
	"fmt"
	"math"
)

type HyperLogLog struct {
	m         uint    // Number of registers
	b         uint32  // Number of bits used to determine register index
	alpha     float64 // Bias correction constant
	registers []uint8
}

// Compute bias correction alpha_m.
func get_alpha(m uint) (result float64) {
	switch m {
	case 16:
		result = 0.673
	case 32:
		result = 0.697
	case 64:
		result = 0.709
	default:
		result = 0.7213 / (1.0 + 1.079/float64(m))
	}
	return result
}

// Return a new HyperLogLog with the given number of registers. More
// registers leads to lower error in your estimated count, at the
// expense of memory.
//
// Choose a power of two number of registers, depending on the amount
// of memory you're willing to use and the error you're willing to
// tolerate. Each register uses one byte of memory.
//
// Approximate error will be:
//     1.04 / sqrt(registers)
//
func New(registers uint) (*HyperLogLog, error) {
	if registers%2 != 0 {
		return nil, fmt.Errorf("number of registers %d not a power of two", registers)
	}
	h := &HyperLogLog{}
	h.m = registers
	h.b = uint32(math.Ceil(math.Log2(float64(registers))))
	h.alpha = get_alpha(registers)
	h.Reset()
	return h, nil
}

// Reset all internal variables and set the count to zero.
func (h *HyperLogLog) Reset() {
	h.registers = make([]uint8, h.m)
}

// Calculate the position of the rightmost 1-bit.
func rho(val uint32, max uint32) uint8 {
	r := 1
	for val&1 == 0 && val <= max {
		r++
		val >>= 1
	}
	return uint8(r)
}

// Add to the count. val should be a 32 bit integer from a good hash
// function.
func (h *HyperLogLog) Add(val uint32) {
	k := 32 - h.b
	r := rho(val, k)
	j := val >> uint(k)
	if r > h.registers[j] {
		h.registers[j] = r
	}
}

// Get the estimated count.
func (h *HyperLogLog) Count() uint64 {
	sum := 0.0
	for _, val := range h.registers {
		sum += 1 / math.Pow(2, float64(val))
	}
	estimate := h.alpha * float64(h.m*h.m) / sum
	pow_2_32 := math.Pow(2, 32)
	if estimate <= 5.0/2.0*float64(h.m) {
		// Small range correction
		v := 0
		for _, r := range h.registers {
			if r == 0 {
				v++
			}
		}
		if v > 0 {
			estimate = float64(h.m) * math.Log(float64(h.m)/float64(v))
		}
	} else if estimate > 1.0/30.0*pow_2_32 {
		// Large range correction
		estimate = -pow_2_32 * math.Log(1-estimate/pow_2_32)
	}
	return uint64(estimate)
}
<|endoftext|>