<commit_before>package iptables

// This package is originally from Docker and has been modified for use by the
// Flynn project. See the NOTICE and LICENSE files for licensing and copyright
// details.

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Action string

const (
	Add    Action = "-A"
	Delete Action = "-D"
)

var (
	ErrIptablesNotFound = errors.New("Iptables not found")
	nat                 = []string{"-t", "nat"}
	supportsXlock       = false
)

type Chain struct {
	Name   string
	Bridge string
}

func init() {
	supportsXlock = exec.Command("iptables", "--wait", "-L", "-n").Run() == nil
}

func NewChain(name, bridge string) (*Chain, error) {
	if output, err := Raw("-t", "nat", "-N", name); err != nil {
		return nil, err
	} else if len(output) != 0 {
		return nil, fmt.Errorf("Error creating new iptables chain: %s", output)
	}
	chain := &Chain{
		Name:   name,
		Bridge: bridge,
	}

	if err := chain.Prerouting(Add, "-m", "addrtype", "--dst-type", "LOCAL"); err != nil {
		return nil, fmt.Errorf("Failed to inject update PREROUTING chain: %s", err)
	}
	if err := chain.Output(Add, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8"); err != nil {
		return nil, fmt.Errorf("Failed to inject update OUTPUT chain: %s", err)
	}
	return chain, nil
}

func RemoveExistingChain(name string) error {
	chain := &Chain{
		Name: name,
	}
	return chain.Remove()
}

func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr string, destPort int) error {
	daddr := ip.String()
	if ip.IsUnspecified() {
		// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
		// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
		// value" by both iptables and ip6tables.
		daddr = "0/0"
	}
	if output, err := Raw("-t", "nat", fmt.Sprint(action), c.Name,
		"-p", proto,
		"-d", daddr,
		"--dport", strconv.Itoa(port),
		"-j", "DNAT",
		"--to-destination", net.JoinHostPort(destAddr, strconv.Itoa(destPort))); err != nil && action != Delete {
		return err
	} else if len(output) != 0 && action != Delete {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	fAction := action
	if fAction == Add {
		fAction = "-I"
	}
	if output, err := Raw(string(fAction), "FORWARD",
		"!", "-i", c.Bridge,
		"-o", c.Bridge,
		"-p", proto,
		"-d", destAddr,
		"--dport", strconv.Itoa(destPort),
		"-j", "ACCEPT"); err != nil && action != Delete {
		return err
	} else if len(output) != 0 && action != Delete {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	if output, err := Raw("-t", "nat", string(fAction), "POSTROUTING",
		"-p", proto,
		"-s", destAddr,
		"-d", destAddr,
		"--dport", strconv.Itoa(destPort),
		"-j", "MASQUERADE"); err != nil && action != Delete {
		return err
	} else if len(output) != 0 && action != Delete {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	return nil
}

func (c *Chain) Prerouting(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "PREROUTING")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables prerouting: %s", output)
	}
	return nil
}

func (c *Chain) Output(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "OUTPUT")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables output: %s", output)
	}
	return nil
}

func (c *Chain) Remove() error {
	// Ignore errors - This could mean the chains were never set up
	c.Prerouting(Delete, "-m", "addrtype", "--dst-type", "LOCAL")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL") // Created in versions <= 0.1.6

	c.Prerouting(Delete)
	c.Output(Delete)

	Raw("-t", "nat", "-F", c.Name)
	Raw("-t", "nat", "-X", c.Name)

	return nil
}

// Check if an existing rule exists
func Exists(args ...string) bool {
	if _, err := Raw(append([]string{"-C"}, args...)...); err != nil {
		return false
	}
	return true
}

func Raw(args ...string) ([]byte, error) {
	path, err := exec.LookPath("iptables")
	if err != nil {
		return nil, ErrIptablesNotFound
	}

	if supportsXlock {
		args = append([]string{"--wait"}, args...)
	}

	if os.Getenv("DEBUG") != "" {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s, %v\n", path, args))
	}

	output, err := exec.Command(path, args...).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err)
	}

	// ignore iptables' message about xtables lock
	if strings.Contains(string(output), "waiting for it to exit") {
		output = []byte("")
	}

	return output, err
}
<commit_msg>Add support for forwarding traffic to localhost<commit_after>package iptables

// This package is originally from Docker and has been modified for use by the
// Flynn project. See the NOTICE and LICENSE files for licensing and copyright
// details.

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Action string

const (
	Add    Action = "-A"
	Delete Action = "-D"
)

var (
	ErrIptablesNotFound = errors.New("Iptables not found")
	nat                 = []string{"-t", "nat"}
	supportsXlock       = false
)

type Chain struct {
	Name   string
	Bridge string
}

func init() {
	supportsXlock = exec.Command("iptables", "--wait", "-L", "-n").Run() == nil
}

func NewChain(name, bridge string) (*Chain, error) {
	if output, err := Raw("-t", "nat", "-N", name); err != nil {
		return nil, err
	} else if len(output) != 0 {
		return nil, fmt.Errorf("Error creating new iptables chain: %s", output)
	}
	chain := &Chain{
		Name:   name,
		Bridge: bridge,
	}

	if err := chain.Prerouting(Add, "-m", "addrtype", "--dst-type", "LOCAL"); err != nil {
		return nil, fmt.Errorf("failed to update PREROUTING chain: %s", err)
	}
	if err := chain.Output(Add, "-m", "addrtype", "--dst-type", "LOCAL"); err != nil {
		return nil, fmt.Errorf("failed to update OUTPUT chain: %s", err)
	}
	if _, err := Raw("-t", "nat", "-A", "POSTROUTING", "-m", "addrtype", "--src-type", "LOCAL", "-o", bridge, "-j", "MASQUERADE"); err != nil {
		return nil, fmt.Errorf("failed to update POSTROUTING chain: %s", err)
	}
	return chain, nil
}

func RemoveExistingChain(name string) error {
	chain := &Chain{
		Name: name,
	}
	return chain.Remove()
}

func (c *Chain) Forward(action Action, ip net.IP, port int, proto, destAddr string, destPort int) error {
	daddr := ip.String()
	if ip.IsUnspecified() {
		// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
		// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
		// value" by both iptables and ip6tables.
		daddr = "0/0"
	}
	if output, err := Raw("-t", "nat", fmt.Sprint(action), c.Name,
		"-p", proto,
		"-d", daddr,
		"--dport", strconv.Itoa(port),
		"-j", "DNAT",
		"--to-destination", net.JoinHostPort(destAddr, strconv.Itoa(destPort))); err != nil && action != Delete {
		return err
	} else if len(output) != 0 && action != Delete {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	fAction := action
	if fAction == Add {
		fAction = "-I"
	}
	if output, err := Raw(string(fAction), "FORWARD",
		"!", "-i", c.Bridge,
		"-o", c.Bridge,
		"-p", proto,
		"-d", destAddr,
		"--dport", strconv.Itoa(destPort),
		"-j", "ACCEPT"); err != nil && action != Delete {
		return err
	} else if len(output) != 0 && action != Delete {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	if output, err := Raw("-t", "nat", string(fAction), "POSTROUTING",
		"-p", proto,
		"-s", destAddr,
		"-d", destAddr,
		"--dport", strconv.Itoa(destPort),
		"-j", "MASQUERADE"); err != nil && action != Delete {
		return err
	} else if len(output) != 0 && action != Delete {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	return nil
}

func (c *Chain) Prerouting(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "PREROUTING")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables prerouting: %s", output)
	}
	return nil
}

func (c *Chain) Output(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "OUTPUT")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables output: %s", output)
	}
	return nil
}

func (c *Chain) Remove() error {
	// Ignore errors - This could mean the chains were never set up
	c.Prerouting(Delete, "-m", "addrtype", "--dst-type", "LOCAL")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL")
	Raw("-t", "nat", "-D", "POSTROUTING", "-m", "addrtype", "--src-type", "LOCAL", "-o", c.Bridge, "-j", "MASQUERADE")

	c.Prerouting(Delete)
	c.Output(Delete)

	Raw("-t", "nat", "-F", c.Name)
	Raw("-t", "nat", "-X", c.Name)

	return nil
}

// Check if an existing rule exists
func Exists(args ...string) bool {
	if _, err := Raw(append([]string{"-C"}, args...)...); err != nil {
		return false
	}
	return true
}

func Raw(args ...string) ([]byte, error) {
	path, err := exec.LookPath("iptables")
	if err != nil {
		return nil, ErrIptablesNotFound
	}

	if supportsXlock {
		args = append([]string{"--wait"}, args...)
	}

	if os.Getenv("DEBUG") != "" {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s, %v\n", path, args))
	}

	output, err := exec.Command(path, args...).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err)
	}

	// ignore iptables' message about xtables lock
	if strings.Contains(string(output), "waiting for it to exit") {
		output = []byte("")
	}

	return output, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

var IPTablesPath = "iptables"

func init() {

	err := CheckIPTables()
	if err != nil {
		wd, err := os.Getwd()
		if err != nil {
			return
		}
		IPTablesPath = filepath.Join(wd, IPTablesPath)
	}

}

// NOTEs from messing with iptables proxying:
// For external:
// iptables -A PREROUTING -t nat -p tcp -m tcp --dport 5555 -j REDIRECT --to-ports 49278
// For internal:
// iptables -A OUTPUT -t nat -p tcp -m tcp --dport 5555 -j REDIRECT --to-ports 49278
// To delete a rule, use -D rather than -A.

type Action bool

const (
	INSERT Action = true
	DELETE        = false
)

func CheckIPTables() error {
	return exec.Command(IPTablesPath, "-L").Run()
}

// Invoke one iptables command.
// Expects "iptables" in the path to be runnable with reasonable permissions.
func iptables(action Action, chain string, source, target int, ipAddress string) *exec.Cmd {
	var cmd *exec.Cmd

	switch action {
	case INSERT:
		cmd = exec.Command(
			IPTablesPath, "--insert", chain, "1",
			"--table", "nat",
			"--protocol", "tcp",
			// Prevent redirection of packets already going to the container
			"--match", "tcp", "!", "--destination", ipAddress,
			// Prevent redirection of ports on remote servers
			// (i.e, don't make google:80 hit our container)
			"--match", "addrtype", "--dst-type", "LOCAL",
			"--dport", fmt.Sprint(source),
			"--jump", "REDIRECT",
			"--to-ports", fmt.Sprint(target))
	case DELETE:
		cmd = exec.Command(
			IPTablesPath, "--delete", chain,
			"--table", "nat",
			"--protocol", "tcp",
			"--match", "tcp", "!", "--destination", ipAddress,
			"--match", "addrtype", "--dst-type", "LOCAL",
			"--dport", fmt.Sprint(source),
			"--jump", "REDIRECT",
			"--to-ports", fmt.Sprint(target))
	}
	cmd.Stderr = os.Stderr
	return cmd
}

// Configure one port redirect from `source` to `target` using iptables.
// Returns an error and a function which undoes the change to the firewall.
func ConfigureRedirect(source, target int, ipAddress string) (func(), error) {

	err := iptables(INSERT, "PREROUTING", source, target, ipAddress).Run()
	if err != nil {
		return nil, err
	}
	err = iptables(INSERT, "OUTPUT", source, target, ipAddress).Run()
	if err != nil {
		return nil, err
	}

	remove := func() {
		err := iptables(DELETE, "PREROUTING", source, target, ipAddress).Run()
		if err != nil {
			log.Println("Failed to remove iptables rule:", source, target)
		}
		err = iptables(DELETE, "OUTPUT", source, target, ipAddress).Run()
		if err != nil {
			log.Println("Failed to remove iptables rule:", source, target)
		}
	}
	return remove, nil
}
<commit_msg>Ensure that iptables waits for a lock<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

var IPTablesPath = "iptables"

func init() {

	err := CheckIPTables()
	if err != nil {
		log.Printf("Unable to find iptables, using fallback")
		wd, err := os.Getwd()
		if err != nil {
			return
		}
		IPTablesPath = filepath.Join(wd, IPTablesPath)
	}

}

// NOTEs from messing with iptables proxying:
// For external:
// iptables -A PREROUTING -t nat -p tcp -m tcp --dport 5555 -j REDIRECT --to-ports 49278
// For internal:
// iptables -A OUTPUT -t nat -p tcp -m tcp --dport 5555 -j REDIRECT --to-ports 49278
// To delete a rule, use -D rather than -A.

type Action bool

const (
	INSERT Action = true
	DELETE        = false
)

func CheckIPTables() error {
	cmd := exec.Command(IPTablesPath, "--list", "--wait")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// Invoke one iptables command.
// Expects "iptables" in the path to be runnable with reasonable permissions.
func iptables(action Action, chain string, source, target int, ipAddress string) *exec.Cmd {
	var cmd *exec.Cmd

	switch action {
	case INSERT:
		cmd = exec.Command(
			IPTablesPath, "--insert", chain, "1",
			"--table", "nat",
			"--protocol", "tcp",
			// Prevent redirection of packets already going to the container
			"--match", "tcp", "!", "--destination", ipAddress,
			// Prevent redirection of ports on remote servers
			// (i.e, don't make google:80 hit our container)
			"--match", "addrtype", "--dst-type", "LOCAL",
			"--dport", fmt.Sprint(source),
			"--jump", "REDIRECT",
			"--to-ports", fmt.Sprint(target), "--wait")
	case DELETE:
		cmd = exec.Command(
			IPTablesPath, "--delete", chain,
			"--table", "nat",
			"--protocol", "tcp",
			"--match", "tcp", "!", "--destination", ipAddress,
			"--match", "addrtype", "--dst-type", "LOCAL",
			"--dport", fmt.Sprint(source),
			"--jump", "REDIRECT",
			"--to-ports", fmt.Sprint(target), "--wait")
	}
	cmd.Stderr = os.Stderr
	return cmd
}

// Configure one port redirect from `source` to `target` using iptables.
// Returns an error and a function which undoes the change to the firewall.
func ConfigureRedirect(source, target int, ipAddress string) (func(), error) {

	err := iptables(INSERT, "PREROUTING", source, target, ipAddress).Run()
	if err != nil {
		return nil, err
	}
	err = iptables(INSERT, "OUTPUT", source, target, ipAddress).Run()
	if err != nil {
		return nil, err
	}

	remove := func() {
		err := iptables(DELETE, "PREROUTING", source, target, ipAddress).Run()
		if err != nil {
			log.Println("Failed to remove iptables rule:", source, target)
		}
		err = iptables(DELETE, "OUTPUT", source, target, ipAddress).Run()
		if err != nil {
			log.Println("Failed to remove iptables rule:", source, target)
		}
	}
	return remove, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package commands

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"text/tabwriter"

	"github.com/romana/core/cli/util"
	"github.com/romana/core/common"
	"github.com/romana/core/common/api"

	"github.com/go-resty/resty"
	ms "github.com/mitchellh/mapstructure"
	cli "github.com/spf13/cobra"
	config "github.com/spf13/viper"
)

// topologyCmd represents the topology commands
var topologyCmd = &cli.Command{
	Use:   "topology [update|list]",
	Short: "Update or List topology for romana services.",
	Long: `Update or List topology for romana services.

topology requires a subcommand, e.g. ` + "`romana topology list`." + `

For more information, please check http://docs.romana.io
`,
}

func init() {
	topologyCmd.AddCommand(topologyListCmd)
	topologyCmd.AddCommand(topologyUpdateCmd)
}

var topologyListCmd = &cli.Command{
	Use:          "list",
	Short:        "List romana topology.",
	Long:         `List romana topology.`,
	RunE:         topologyList,
	SilenceUsage: true,
}

var topologyUpdateCmd = &cli.Command{
	Use:          "update [file name]",
	Short:        "Update romana topology.",
	Long:         `Update romana topology.`,
	RunE:         topologyUpdate,
	SilenceUsage: true,
}

func topologyList(cmd *cli.Command, args []string) error {
	rootURL := config.GetString("RootURL")
	resp, err := resty.R().Get(rootURL + "/topology")
	if err != nil {
		return err
	}

	if config.GetString("Format") == "json" {
		JSONFormat(resp.Body(), os.Stdout)
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)

		if resp.StatusCode() == http.StatusOK {
			var topology api.TopologyUpdateRequest
			err := json.Unmarshal(resp.Body(), &topology)
			if err == nil {
				fmt.Println("Networks")
				fmt.Fprint(w, "Name\tCIDR\tTenants\n")
				for _, n := range topology.Networks {
					fmt.Fprintf(w, "%s\t%s\t%v\n",
						n.Name,
						n.CIDR,
						n.Tenants,
					)
				}
				fmt.Fprint(w, "\n")
				for _, t := range topology.Topologies {
					fmt.Printf("Topology for Network/s: %s\n", t.Networks)
					fmt.Fprint(w, "Name\tCIDR\tNodes\n")
					for _, m := range t.Map {
						fmt.Fprintf(w, "%s\t%s\t", m.Name, m.CIDR)
						for _, n := range m.Groups {
							fmt.Fprintf(w, "%s(%s), ", n.Name, n.IP)
						}
						fmt.Fprint(w, "\n")
					}
					fmt.Fprint(w, "\n")
				}
			} else {
				fmt.Printf("Error: %s \n", err)
			}
		} else {
			var e Error
			json.Unmarshal(resp.Body(), &e)

			fmt.Println("Host Error")
			fmt.Fprintf(w, "Fields\t%s\n", e.Fields)
			fmt.Fprintf(w, "Message\t%s\n", e.Message)
			fmt.Fprintf(w, "Status\t%d\n", resp.StatusCode())
		}
		w.Flush()
	}

	return nil
}

// topologyUpdate updates romana topology.
// The features supported are:
//  * Topology update through file
//  * Topology update while taking input from standard
//    input (STDIN) instead of a file
func topologyUpdate(cmd *cli.Command, args []string) error {
	var buf []byte
	var err error
	isFile := true

	if len(args) == 0 {
		isFile = false
		buf, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			util.UsageError(cmd,
				"TOPOLOGY FILE name or piped input from 'STDIN' expected.")
			return fmt.Errorf("cannot read 'STDIN': %s", err)
		}
	} else if len(args) != 1 {
		return util.UsageError(cmd,
			"TOPOLOGY FILE name or piped input from 'STDIN' expected.")
	}

	rootURL := config.GetString("RootURL")

	var topology api.TopologyUpdateRequest
	if isFile {
		pBuf, err := ioutil.ReadFile(args[0])
		if err != nil {
			return fmt.Errorf("file error: %s", err)
		}
		err = json.Unmarshal(pBuf, &topology)
		if err != nil {
			return err
		}
	} else {
		err = json.Unmarshal(buf, &topology)
		if err != nil {
			return err
		}
	}

	resp, err := resty.R().SetHeader("Content-Type", "application/json").
		SetBody(topology).Post(rootURL + "/topology")
	if err != nil {
		log.Printf("Error updating topology: %v\n", err)
		return err
	}

	if config.GetString("Format") == "json" {
		if string(resp.Body()) == "" || string(resp.Body()) == "null" {
			var h common.HttpError
			dc := &ms.DecoderConfig{TagName: "json", Result: &h}
			decoder, err := ms.NewDecoder(dc)
			if err != nil {
				return err
			}
			m := make(map[string]interface{})
			m["details"] = resp.Status()
			m["status_code"] = resp.StatusCode()
			err = decoder.Decode(m)
			if err != nil {
				return err
			}
			status, _ := json.MarshalIndent(h, "", "\t")
			fmt.Println(string(status))
		} else {
			JSONFormat(resp.Body(), os.Stdout)
		}
	} else {
		if resp.StatusCode() == http.StatusOK {
			fmt.Println("Topology updated successfully.")
		} else {
			fmt.Printf("Error upadting topology: %s\n", resp.Status())
		}
	}

	return nil
}
<commit_msg>cli: remove extraneous error message.<commit_after>// Copyright (c) 2018 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package commands

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"text/tabwriter"

	"github.com/romana/core/cli/util"
	"github.com/romana/core/common"
	"github.com/romana/core/common/api"

	"github.com/go-resty/resty"
	ms "github.com/mitchellh/mapstructure"
	cli "github.com/spf13/cobra"
	config "github.com/spf13/viper"
)

// topologyCmd represents the topology commands
var topologyCmd = &cli.Command{
	Use:   "topology [update|list]",
	Short: "Update or List topology for romana services.",
	Long: `Update or List topology for romana services.

topology requires a subcommand, e.g. ` + "`romana topology list`." + `

For more information, please check http://docs.romana.io
`,
}

func init() {
	topologyCmd.AddCommand(topologyListCmd)
	topologyCmd.AddCommand(topologyUpdateCmd)
}

var topologyListCmd = &cli.Command{
	Use:          "list",
	Short:        "List romana topology.",
	Long:         `List romana topology.`,
	RunE:         topologyList,
	SilenceUsage: true,
}

var topologyUpdateCmd = &cli.Command{
	Use:          "update [file name]",
	Short:        "Update romana topology.",
	Long:         `Update romana topology.`,
	RunE:         topologyUpdate,
	SilenceUsage: true,
}

func topologyList(cmd *cli.Command, args []string) error {
	rootURL := config.GetString("RootURL")
	resp, err := resty.R().Get(rootURL + "/topology")
	if err != nil {
		return err
	}

	if config.GetString("Format") == "json" {
		JSONFormat(resp.Body(), os.Stdout)
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)

		if resp.StatusCode() == http.StatusOK {
			var topology api.TopologyUpdateRequest
			err := json.Unmarshal(resp.Body(), &topology)
			if err == nil {
				fmt.Println("Networks")
				fmt.Fprint(w, "Name\tCIDR\tTenants\n")
				for _, n := range topology.Networks {
					fmt.Fprintf(w, "%s\t%s\t%v\n",
						n.Name,
						n.CIDR,
						n.Tenants,
					)
				}
				fmt.Fprint(w, "\n")
				for _, t := range topology.Topologies {
					fmt.Printf("Topology for Network/s: %s\n", t.Networks)
					fmt.Fprint(w, "Name\tCIDR\tNodes\n")
					for _, m := range t.Map {
						fmt.Fprintf(w, "%s\t%s\t", m.Name, m.CIDR)
						for _, n := range m.Groups {
							fmt.Fprintf(w, "%s(%s), ", n.Name, n.IP)
						}
						fmt.Fprint(w, "\n")
					}
					fmt.Fprint(w, "\n")
				}
			} else {
				fmt.Printf("Error: %s \n", err)
			}
		} else {
			var e Error
			json.Unmarshal(resp.Body(), &e)

			fmt.Println("Host Error")
			fmt.Fprintf(w, "Fields\t%s\n", e.Fields)
			fmt.Fprintf(w, "Message\t%s\n", e.Message)
			fmt.Fprintf(w, "Status\t%d\n", resp.StatusCode())
		}
		w.Flush()
	}

	return nil
}

// topologyUpdate updates romana topology.
// The features supported are:
//  * Topology update through file
//  * Topology update while taking input from standard
//    input (STDIN) instead of a file
func topologyUpdate(cmd *cli.Command, args []string) error {
	var buf []byte
	var err error
	isFile := true

	if len(args) == 0 {
		isFile = false
		buf, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			return fmt.Errorf("cannot read 'STDIN': %s", err)
		}
	} else if len(args) != 1 {
		return util.UsageError(cmd,
			"TOPOLOGY FILE name or piped input from 'STDIN' expected.")
	}

	rootURL := config.GetString("RootURL")

	var topology api.TopologyUpdateRequest
	if isFile {
		pBuf, err := ioutil.ReadFile(args[0])
		if err != nil {
			return fmt.Errorf("file error: %s", err)
		}
		err = json.Unmarshal(pBuf, &topology)
		if err != nil {
			return err
		}
	} else {
		err = json.Unmarshal(buf, &topology)
		if err != nil {
			return err
		}
	}

	resp, err := resty.R().SetHeader("Content-Type", "application/json").
		SetBody(topology).Post(rootURL + "/topology")
	if err != nil {
		log.Printf("Error updating topology: %v\n", err)
		return err
	}

	if config.GetString("Format") == "json" {
		if string(resp.Body()) == "" || string(resp.Body()) == "null" {
			var h common.HttpError
			dc := &ms.DecoderConfig{TagName: "json", Result: &h}
			decoder, err := ms.NewDecoder(dc)
			if err != nil {
				return err
			}
			m := make(map[string]interface{})
			m["details"] = resp.Status()
			m["status_code"] = resp.StatusCode()
			err = decoder.Decode(m)
			if err != nil {
				return err
			}
			status, _ := json.MarshalIndent(h, "", "\t")
			fmt.Println(string(status))
		} else {
			JSONFormat(resp.Body(), os.Stdout)
		}
	} else {
		if resp.StatusCode() == http.StatusOK {
			fmt.Println("Topology updated successfully.")
		} else {
			fmt.Printf("Error upadting topology: %s\n", resp.Status())
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

//go:generate go run gen.go

// This program generates internet protocol constants and tables by
// reading IANA protocol registries.
package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
)

var registries = []struct {
	url   string
	parse func(io.Writer, io.Reader) error
}{
	{
		"http://www.iana.org/assignments/icmp-parameters/icmp-parameters.xml",
		parseICMPv4Parameters,
	},
}

func main() {
	var bb bytes.Buffer
	fmt.Fprintf(&bb, "// go generate gen.go\n")
	fmt.Fprintf(&bb, "// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT\n\n")
	fmt.Fprintf(&bb, "package ipv4\n\n")
	for _, r := range registries {
		resp, err := http.Get(r.url)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			fmt.Fprintf(os.Stderr, "got HTTP status code %v for %v\n", resp.StatusCode, r.url)
			os.Exit(1)
		}
		if err := r.parse(&bb, resp.Body); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		fmt.Fprintf(&bb, "\n")
	}
	b, err := format.Source(bb.Bytes())
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if err := ioutil.WriteFile("iana.go", b, 0644); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func parseICMPv4Parameters(w io.Writer, r io.Reader) error {
	dec := xml.NewDecoder(r)
	var icp icmpv4Parameters
	if err := dec.Decode(&icp); err != nil {
		return err
	}
	prs := icp.escape()
	fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated)
	fmt.Fprintf(w, "const (\n")
	for _, pr := range prs {
		if pr.Descr == "" {
			continue
		}
		fmt.Fprintf(w, "ICMPType%s ICMPType = %d", pr.Descr, pr.Value)
		fmt.Fprintf(w, "// %s\n", pr.OrigDescr)
	}
	fmt.Fprintf(w, ")\n\n")
	fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated)
	fmt.Fprintf(w, "var icmpTypes = map[ICMPType]string{\n")
	for _, pr := range prs {
		if pr.Descr == "" {
			continue
		}
		fmt.Fprintf(w, "%d: %q,\n", pr.Value, strings.ToLower(pr.OrigDescr))
	}
	fmt.Fprintf(w, "}\n")
	return nil
}

type icmpv4Parameters struct {
	XMLName    xml.Name `xml:"registry"`
	Title      string   `xml:"title"`
	Updated    string   `xml:"updated"`
	Registries []struct {
		Title   string `xml:"title"`
		Records []struct {
			Value string `xml:"value"`
			Descr string `xml:"description"`
		} `xml:"record"`
	} `xml:"registry"`
}

type canonICMPv4ParamRecord struct {
	OrigDescr string
	Descr     string
	Value     int
}

func (icp *icmpv4Parameters) escape() []canonICMPv4ParamRecord {
	id := -1
	for i, r := range icp.Registries {
		if strings.Contains(r.Title, "Type") || strings.Contains(r.Title, "type") {
			id = i
			break
		}
	}
	if id < 0 {
		return nil
	}
	prs := make([]canonICMPv4ParamRecord, len(icp.Registries[id].Records))
	sr := strings.NewReplacer(
		"Messages", "",
		"Message", "",
		"ICMP", "",
		"+", "P",
		"-", "",
		"/", "",
		".", "",
		" ", "",
	)
	for i, pr := range icp.Registries[id].Records {
		if strings.Contains(pr.Descr, "Reserved") ||
			strings.Contains(pr.Descr, "Unassigned") ||
			strings.Contains(pr.Descr, "Deprecated") ||
			strings.Contains(pr.Descr, "Experiment") ||
			strings.Contains(pr.Descr, "experiment") {
			continue
		}
		ss := strings.Split(pr.Descr, "\n")
		if len(ss) > 1 {
			prs[i].Descr = strings.Join(ss, " ")
		} else {
			prs[i].Descr = ss[0]
		}
		s := strings.TrimSpace(prs[i].Descr)
		prs[i].OrigDescr = s
		prs[i].Descr = sr.Replace(s)
		prs[i].Value, _ = strconv.Atoi(pr.Value)
	}
	return prs
}
<commit_msg>go.net/ipv4: make use of go generate to create system adaptation files<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

//go:generate go run gen.go

// This program generates system adaptation constants and types,
// internet protocol constants and tables by reading template files
// and IANA protocol registries.
package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
)

func main() {
	if err := genzsys(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if err := geniana(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func genzsys() error {
	defs := "defs_" + runtime.GOOS + ".go"
	f, err := os.Open(defs)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	f.Close()
	cmd := exec.Command("go", "tool", "cgo", "-godefs", defs)
	b, err := cmd.Output()
	if err != nil {
		return err
	}
	switch runtime.GOOS {
	case "dragonfly", "solaris":
		// The ipv4 pacakge still supports go1.2, and so we
		// need to take care of additional platforms in go1.3
		// and above for working with go1.2.
		b = bytes.Replace(b, []byte("package ipv4\n"), []byte("// +build "+runtime.GOOS+"\n\npackage ipv4\n"), 1)
	}
	b, err = format.Source(b)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile("zsys_"+runtime.GOOS+".go", b, 0644); err != nil {
		return err
	}
	return nil
}

var registries = []struct {
	url   string
	parse func(io.Writer, io.Reader) error
}{
	{
		"http://www.iana.org/assignments/icmp-parameters/icmp-parameters.xml",
		parseICMPv4Parameters,
	},
}

func geniana() error {
	var bb bytes.Buffer
	fmt.Fprintf(&bb, "// go generate gen.go\n")
	fmt.Fprintf(&bb, "// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT\n\n")
	fmt.Fprintf(&bb, "package ipv4\n\n")
	for _, r := range registries {
		resp, err := http.Get(r.url)
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("got HTTP status code %v for %v\n", resp.StatusCode, r.url)
		}
		if err := r.parse(&bb, resp.Body); err != nil {
			return err
		}
		fmt.Fprintf(&bb, "\n")
	}
	b, err := format.Source(bb.Bytes())
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile("iana.go", b, 0644); err != nil {
		return err
	}
	return nil
}

func parseICMPv4Parameters(w io.Writer, r io.Reader) error {
	dec := xml.NewDecoder(r)
	var icp icmpv4Parameters
	if err := dec.Decode(&icp); err != nil {
		return err
	}
	prs := icp.escape()
	fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated)
	fmt.Fprintf(w, "const (\n")
	for _, pr := range prs {
		if pr.Descr == "" {
			continue
		}
		fmt.Fprintf(w, "ICMPType%s ICMPType = %d", pr.Descr, pr.Value)
		fmt.Fprintf(w, "// %s\n", pr.OrigDescr)
	}
	fmt.Fprintf(w, ")\n\n")
	fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated)
	fmt.Fprintf(w, "var icmpTypes = map[ICMPType]string{\n")
	for _, pr := range prs {
		if pr.Descr == "" {
			continue
		}
		fmt.Fprintf(w, "%d: %q,\n", pr.Value, strings.ToLower(pr.OrigDescr))
	}
	fmt.Fprintf(w, "}\n")
	return nil
}

type icmpv4Parameters struct {
	XMLName    xml.Name `xml:"registry"`
	Title      string   `xml:"title"`
	Updated    string   `xml:"updated"`
	Registries []struct {
		Title   string `xml:"title"`
		Records []struct {
			Value string `xml:"value"`
			Descr string `xml:"description"`
		} `xml:"record"`
	} `xml:"registry"`
}

type canonICMPv4ParamRecord struct {
	OrigDescr string
	Descr     string
	Value     int
}

func (icp *icmpv4Parameters) escape() []canonICMPv4ParamRecord {
	id := -1
	for i, r := range icp.Registries {
		if strings.Contains(r.Title, "Type") || strings.Contains(r.Title, "type") {
			id = i
			break
		}
	}
	if id < 0 {
		return nil
	}
	prs := make([]canonICMPv4ParamRecord, len(icp.Registries[id].Records))
	sr := strings.NewReplacer(
		"Messages", "",
		"Message", "",
		"ICMP", "",
		"+", "P",
		"-", "",
		"/", "",
		".", "",
		" ", "",
	)
	for i, pr := range icp.Registries[id].Records {
		if strings.Contains(pr.Descr, "Reserved") ||
			strings.Contains(pr.Descr, "Unassigned") ||
			strings.Contains(pr.Descr, "Deprecated") ||
			strings.Contains(pr.Descr, "Experiment") ||
			strings.Contains(pr.Descr, "experiment") {
			continue
		}
		ss := strings.Split(pr.Descr, "\n")
		if len(ss) > 1 {
			prs[i].Descr = strings.Join(ss, " ")
		} else {
			prs[i].Descr = ss[0]
		}
		s := strings.TrimSpace(prs[i].Descr)
		prs[i].OrigDescr = s
		prs[i].Descr = sr.Replace(s)
		prs[i].Value, _ = strconv.Atoi(pr.Value)
	}
	return prs
}
<|endoftext|>


<commit_before>package registry

import (
	"context"
	"encoding/json"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/registry/cache"
)

// A client represents an entity that returns manifest and tags
// information.  It might be a cache, it might be a real registry.
type Client interface {
	Tags(id flux.ImageID) ([]string, error)
	Manifest(id flux.ImageID) (flux.Image, error)
	Cancel()
}

// ---

// An implementation of Client that represents a Remote registry.
// E.g. docker hub.
type Remote struct {
	Registry   HerokuRegistryLibrary
	CancelFunc context.CancelFunc
}

// Return the tags for this repository.
func (a *Remote) Tags(id flux.ImageID) ([]string, error) {
	return a.Registry.Tags(id.NamespaceImage())
}

// We need to do some adapting here to convert from the return values
// from dockerregistry to our domain types.
func (a *Remote) Manifest(id flux.ImageID) (flux.Image, error) {
	history, err := a.Registry.Manifest(id.NamespaceImage(), id.Tag)
	if err != nil || history == nil {
		return flux.Image{}, errors.Wrap(err, "getting remote manifest")
	}

	// the manifest includes some v1-backwards-compatibility data,
	// oddly called "History", which are layer metadata as JSON
	// strings; these appear most-recent (i.e., topmost layer) first,
	// so happily we can just decode the first entry to get a created
	// time.
	type v1image struct {
		Created time.Time `json:"created"`
	}
	var topmost v1image
	var img flux.Image
	img.ID = id
	if len(history) > 0 {
		if err = json.Unmarshal([]byte(history[0].V1Compatibility), &topmost); err == nil {
			if !topmost.Created.IsZero() {
				img.CreatedAt = topmost.Created
			}
		}
	}

	return img, nil
}

// Cancel the remote request
func (a *Remote) Cancel() {
	a.CancelFunc()
}

// ---

// An implementation of Client backed by Memcache
type Cache struct {
	creds  Credentials
	expiry time.Duration
	cr     cache.Reader
	logger log.Logger
}

func (*Cache) Cancel() {
	return
}

func NewCache(creds Credentials, cr cache.Reader, expiry time.Duration, logger log.Logger) Client {
	return &Cache{
		creds:  creds,
		expiry: expiry,
		cr:     cr,
		logger: logger,
	}
}

func (c *Cache) Manifest(id flux.ImageID) (flux.Image, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewManifestKey(creds.username, id)
	if err != nil {
		return flux.Image{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return flux.Image{}, err
	}
	var img flux.Image
	err = json.Unmarshal(val, &img)
	if err != nil {
		c.logger.Log("err", err.Error)
		return flux.Image{}, err
	}
	return img, nil
}

func (c *Cache) Tags(id flux.ImageID) ([]string, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewTagKey(creds.username, id)
	if err != nil {
		return []string{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return []string{}, err
	}
	var tags []string
	err = json.Unmarshal(val, &tags)
	if err != nil {
		c.logger.Log("err", err.Error)
		return []string{}, err
	}
	return tags, nil
}
<commit_msg>Try to get a schema2 manifest first<commit_after>package registry

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/go-kit/kit/log"
	dockerregistry "github.com/heroku/docker-registry-client/registry"
	"github.com/pkg/errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/registry/cache"
)

// A client represents an entity that returns manifest and tags
// information.  It might be a cache, it might be a real registry.
type Client interface {
	Tags(id flux.ImageID) ([]string, error)
	Manifest(id flux.ImageID) (flux.Image, error)
	Cancel()
}

// ---

// An implementation of Client that represents a Remote registry.
// E.g. docker hub.
type Remote struct {
	Registry   *herokuManifestAdaptor
	CancelFunc context.CancelFunc
}

// Return the tags for this repository.
func (a *Remote) Tags(id flux.ImageID) ([]string, error) {
	return a.Registry.Tags(id.NamespaceImage())
}

// We need to do some adapting here to convert from the return values
// from dockerregistry to our domain types.
func (a *Remote) Manifest(id flux.ImageID) (flux.Image, error) {
	manifestV2, err := a.Registry.ManifestV2(id.NamespaceImage(), id.Tag)
	if err != nil {
		if err, ok := err.(*dockerregistry.HttpStatusError); ok {
			if err.Response.StatusCode == http.StatusNotFound {
				return a.ManifestFromV1(id)
			}
		}
		return flux.Image{}, err
	}
	// The above request will happily return a bogus, empty manifest
	// if handed something other than a schema2 manifest.
	if manifestV2.Config.Digest == "" {
		return a.ManifestFromV1(id)
	}

	// schema2 manifests have a reference to a blog that contains the
	// image config. We have to fetch that in order to get the created
	// datetime.
	conf := manifestV2.Config
	reader, err := a.Registry.DownloadLayer(id.NamespaceImage(), conf.Digest)
	if err != nil {
		return flux.Image{}, err
	}
	if reader == nil {
		return flux.Image{}, fmt.Errorf("nil reader from DownloadLayer")
	}

	type config struct {
		Created time.Time `json:created`
	}
	var imageConf config

	err = json.NewDecoder(reader).Decode(&imageConf)
	if err != nil {
		return flux.Image{}, err
	}
	return flux.Image{
		ID:        id,
		CreatedAt: imageConf.Created,
	}, nil
}

func (a *Remote) ManifestFromV1(id flux.ImageID) (flux.Image, error) {
	history, err := a.Registry.Manifest(id.NamespaceImage(), id.Tag)
	if err != nil || history == nil {
		return flux.Image{}, errors.Wrap(err, "getting remote manifest")
	}

	// the manifest includes some v1-backwards-compatibility data,
	// oddly called "History", which are layer metadata as JSON
	// strings; these appear most-recent (i.e., topmost layer) first,
	// so happily we can just decode the first entry to get a created
	// time.
	type v1image struct {
		Created time.Time `json:"created"`
	}
	var topmost v1image
	var img flux.Image
	img.ID = id
	if len(history) > 0 {
		if err = json.Unmarshal([]byte(history[0].V1Compatibility), &topmost); err == nil {
			if !topmost.Created.IsZero() {
				img.CreatedAt = topmost.Created
			}
		}
	}

	return img, nil
}

// Cancel the remote request
func (a *Remote) Cancel() {
	a.CancelFunc()
}

// ---

// An implementation of Client backed by Memcache
type Cache struct {
	creds  Credentials
	expiry time.Duration
	cr     cache.Reader
	logger log.Logger
}

func (*Cache) Cancel() {
	return
}

func NewCache(creds Credentials, cr cache.Reader, expiry time.Duration, logger log.Logger) Client {
	return &Cache{
		creds:  creds,
		expiry: expiry,
		cr:     cr,
		logger: logger,
	}
}

func (c *Cache) Manifest(id flux.ImageID) (flux.Image, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewManifestKey(creds.username, id)
	if err != nil {
		return flux.Image{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return flux.Image{}, err
	}
	var img flux.Image
	err = json.Unmarshal(val, &img)
	if err != nil {
		c.logger.Log("err", err.Error)
		return flux.Image{}, err
	}
	return img, nil
}

func (c *Cache) Tags(id flux.ImageID) ([]string, error) {
	creds := c.creds.credsFor(id.Host)
	key, err := cache.NewTagKey(creds.username, id)
	if err != nil {
		return []string{}, err
	}
	val, err := c.cr.GetKey(key)
	if err != nil {
		return []string{}, err
	}
	var tags []string
	err = json.Unmarshal(val, &tags)
	if err != nil {
		c.logger.Log("err", err.Error)
		return []string{}, err
	}
	return tags, nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sacloud

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/sacloud/libsacloud/v2/sacloud/accessor"
	"github.com/sacloud/libsacloud/v2/sacloud/types"
)

// StateWaiter リソースの状態が変わるまで待機する
type StateWaiter interface {
	// WaitForState リソースが指定の状態になるまで待つ
	WaitForState(context.Context) (interface{}, error)
	// AsyncWaitForState リソースが指定の状態になるまで待つ
	AsyncWaitForState(context.Context) (compCh <-chan interface{}, progressCh <-chan interface{}, errorCh <-chan error)
}

var (
	// DefaultStatePollingTimeout StatePollWaiterでのデフォルトタイムアウト
	DefaultStatePollingTimeout = 20 * time.Minute
	// DefaultStatePollingInterval StatePollWaiterでのデフォルトポーリング間隔
	DefaultStatePollingInterval = 5 * time.Second
)

// StateReadFunc StatePollWaiterにより利用される、対象リソースの状態を取得するためのfunc
type StateReadFunc func() (state interface{}, err error)

// StateCheckFunc StateReadFuncで得たリソースの情報を元に待ちを継続するか判定するためのfunc
//
// StatePollWaiterのフィールドとして設定する
type StateCheckFunc func(target interface{}) (exit bool, err error)

// UnexpectedAvailabilityError 予期しないAvailabilityとなった場合のerror
type UnexpectedAvailabilityError struct {
	// Err エラー詳細
	Err error
}

// Error errorインターフェース実装
func (e *UnexpectedAvailabilityError) Error() string {
	return fmt.Sprintf("resource returns unexpected availability value: %s", e.Err.Error())
}

// UnexpectedInstanceStatusError 予期しないInstanceStatusとなった場合のerror
type UnexpectedInstanceStatusError struct {
	// Err エラー詳細
	Err error
}

// Error errorインターフェース実装
func (e *UnexpectedInstanceStatusError) Error() string {
	return fmt.Sprintf("resource returns unexpected instance status value: %s", e.Err.Error())
}

// StatePollingWaiter ポーリングによりリソースの状態が変わるまで待機する
type StatePollingWaiter struct {
	// NotFoundRetry Readで404が返ってきた場合のリトライ回数
	//
	// アプライアンスなどの一部のリソースでは作成~起動完了までの間に404を返すことがある。
	// これに対応するためこのフィールドにて404発生の許容回数を指定可能にする。
	NotFoundRetry int

	// ReadFunc 対象リソースの状態を取得するためのfunc
	//
	// TargetAvailabilityを指定する場合はAvailabilityHolderを返す必要がある
	// もしAvailabilityHolderを実装しておらず、かつStateCheckFuncも未指定だった場合はタイムアウトまで完了しないため注意
	ReadFunc StateReadFunc

	// TargetAvailability 対象リソースのAvailabilityがこの状態になった場合になるまで待つ
	//
	// この値を指定する場合、ReadFuncにてAvailabilityHolderを返す必要がある。
	// AvailabilityがTargetAvailabilityとPendingAvailabilityで指定されていない状態になった場合はUnexpectedAvailabilityErrorを返す
	//
	// TargetAvailability(Pending)とTargetInstanceState(Pending)の両方が指定された場合は両方を満たすまで待つ
	// StateCheckFuncとの併用は不可。併用した場合はpanicする。
	TargetAvailability []types.EAvailability

	// PendingAvailability 対象リソースのAvailabilityがこの状態になった場合は待ちを継続する。
	//
	// 詳細はTargetAvailabilityのコメントを参照
	PendingAvailability []types.EAvailability

	// TargetInstanceStatus 対象リソースのInstanceStatusがこの状態になった場合になるまで待つ
	//
	// この値を指定する場合、ReadFuncにてInstanceStatusHolderを返す必要がある。
	// InstanceStatusがTargetInstanceStatusとPendinngInstanceStatusで指定されていない状態になった場合はUnexpectedInstanceStatusErrorを返す
	//
	// TargetAvailabilityとTargetInstanceStateの両方が指定された場合は両方を満たすまで待つ
	//
	// StateCheckFuncとの併用は不可。併用した場合はpanicする。
	TargetInstanceStatus []types.EServerInstanceStatus

	// PendingInstanceStatus 対象リソースのInstanceStatusがこの状態になった場合は待ちを継続する。
	//
	// 詳細はTargetInstanceStatusのコメントを参照
	PendingInstanceStatus []types.EServerInstanceStatus

	// StateCheckFunc ReadFuncで得たリソースの情報を元に待ちを継続するかの判定を行うためのfunc
	//
	// TargetAvailabilityとTargetInstanceStateとの併用は不可。併用した場合panicする
	StateCheckFunc StateCheckFunc

	// Timeout タイムアウト
	Timeout time.Duration // タイムアウト
	// PollingInterval ポーリング間隔
	PollingInterval time.Duration
}

func (w *StatePollingWaiter) validateFields() {
	if w.ReadFunc == nil {
		panic(errors.New("StatePollingWaiter has invalid setting: ReadFunc is required"))
	}

	if w.StateCheckFunc != nil && (len(w.TargetAvailability) > 0 || len(w.TargetInstanceStatus) > 0) {
		panic(errors.New("StatePollingWaiter has invalid setting: StateCheckFunc and TargetAvailability/TargetInstanceStatus can not use together"))
	}

	if w.StateCheckFunc == nil && len(w.TargetAvailability) == 0 && len(w.TargetInstanceStatus) == 0 {
		panic(errors.New("StatePollingWaiter has invalid setting: TargetAvailability or TargetInstanceState must have least 1 items when StateCheckFunc is not set"))
	}
}

func (w *StatePollingWaiter) defaults() {

	if w.Timeout == time.Duration(0) {
		w.Timeout = DefaultStatePollingTimeout
	}
	if w.PollingInterval == time.Duration(0) {
		w.PollingInterval = DefaultStatePollingInterval
	}
}

// WaitForState リソースが指定の状態になるまで待つ
func (w *StatePollingWaiter) WaitForState(ctx context.Context) (interface{}, error) {
	c, p, e := w.AsyncWaitForState(ctx)
	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case lastState := <-c:
			return lastState, nil
		case <-p:
			// noop
		case err := <-e:
			return nil, err
		}
	}
}

// AsyncWaitForState リソースが指定の状態になるまで待つ
func (w *StatePollingWaiter) AsyncWaitForState(ctx context.Context) (compCh <-chan interface{}, progressCh <-chan interface{}, errorCh <-chan error) {

	w.validateFields()
	w.defaults()

	compChan := make(chan interface{})
	progChan := make(chan interface{})
	errChan := make(chan error)

	ticker := time.NewTicker(w.PollingInterval)

	go func() {
		ctx, cancel := context.WithTimeout(ctx, w.Timeout)
		defer cancel()

		defer ticker.Stop()

		defer close(compChan)
		defer close(progChan)
		defer close(errChan)

		notFoundCounter := w.NotFoundRetry
		for {
			select {
			case <-ctx.Done():
				errChan <- ctx.Err()
				return
			case <-ticker.C:
				state, err := w.ReadFunc()

				if err != nil {
					if IsNotFoundError(err) {
						notFoundCounter--
						if notFoundCounter >= 0 {
							continue
						}
					}
					errChan <- err
					return
				}

				exit, err := w.handleState(state)
				if exit {
					compChan <- state
					return
				}

				if err != nil {
					errChan <- err
					return
				}

				if state != nil {
					progChan <- state
				}
			}
		}
	}()

	compCh = compChan
	progressCh = progChan
	errorCh = errChan
	return
}

func (w *StatePollingWaiter) handleState(state interface{}) (bool, error) {
	if w.StateCheckFunc != nil {
		return w.StateCheckFunc(state)
	}

	availabilityHolder, hasAvailability := state.(accessor.Availability)
	instanceStateHolder, hasInstanceState := state.(accessor.InstanceStatus)

	switch {
	case hasAvailability && hasInstanceState:

		res1, err := w.handleAvailability(availabilityHolder)
		if err != nil {
			return false, err
		}
		res2, err := w.handleInstanceState(instanceStateHolder)
		if err != nil {
			return false, err
		}
		return res1 && res2, nil

	case hasAvailability:
		return w.handleAvailability(availabilityHolder)
	case hasInstanceState:
		return w.handleInstanceState(instanceStateHolder)
	default:
		// どちらのインターフェースも実装していない場合、stateが存在するだけでtrueとする
		return true, nil
	}
}

func (w *StatePollingWaiter) handleAvailability(state accessor.Availability) (bool, error) {
	if len(w.TargetAvailability) == 0 {
		return true, nil
	}
	v := state.GetAvailability()
	switch {
	case w.isInAvailability(v, w.TargetAvailability):
		return true, nil
	case w.isInAvailability(v, w.PendingAvailability):
		return false, nil
	default:
		return false, fmt.Errorf("got unexpected value of Availability: got %q", v)
	}
}

func (w *StatePollingWaiter) handleInstanceState(state accessor.InstanceStatus) (bool, error) {
	if len(w.TargetInstanceStatus) == 0 {
		return true, nil
	}
	v := state.GetInstanceStatus()
	switch {
	case w.isInInstanceStatus(v, w.TargetInstanceStatus):
		return true, nil
	case w.isInInstanceStatus(v, w.PendingInstanceStatus):
		return false, nil
	default:
		return false, fmt.Errorf("got unexpected value of InstanceState: got %q", v)
	}
}

func (w *StatePollingWaiter) isInAvailability(v types.EAvailability, conds []types.EAvailability) bool {
	for _, cond := range conds {
		if v == cond {
			return true
		}
	}
	return false
}

func (w *StatePollingWaiter) isInInstanceStatus(v types.EServerInstanceStatus, conds []types.EServerInstanceStatus) bool {
	for _, cond := range conds {
		if v == cond {
			return true
		}
	}
	return false
}
<commit_msg>lint: nakedret<commit_after>// Copyright 2016-2019 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sacloud

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/sacloud/libsacloud/v2/sacloud/accessor"
	"github.com/sacloud/libsacloud/v2/sacloud/types"
)

// StateWaiter リソースの状態が変わるまで待機する
type StateWaiter interface {
	// WaitForState リソースが指定の状態になるまで待つ
	WaitForState(context.Context) (interface{}, error)
	// AsyncWaitForState リソースが指定の状態になるまで待つ
	AsyncWaitForState(context.Context) (compCh <-chan interface{}, progressCh <-chan interface{}, errorCh <-chan error)
}

var (
	// DefaultStatePollingTimeout StatePollWaiterでのデフォルトタイムアウト
	DefaultStatePollingTimeout = 20 * time.Minute
	// DefaultStatePollingInterval StatePollWaiterでのデフォルトポーリング間隔
	DefaultStatePollingInterval = 5 * time.Second
)

// StateReadFunc StatePollWaiterにより利用される、対象リソースの状態を取得するためのfunc
type StateReadFunc func() (state interface{}, err error)

// StateCheckFunc StateReadFuncで得たリソースの情報を元に待ちを継続するか判定するためのfunc
//
// StatePollWaiterのフィールドとして設定する
type StateCheckFunc func(target interface{}) (exit bool, err error)

// UnexpectedAvailabilityError 予期しないAvailabilityとなった場合のerror
type UnexpectedAvailabilityError struct {
	// Err エラー詳細
	Err error
}

// Error errorインターフェース実装
func (e *UnexpectedAvailabilityError) Error() string {
	return fmt.Sprintf("resource returns unexpected availability value: %s", e.Err.Error())
}

// UnexpectedInstanceStatusError 予期しないInstanceStatusとなった場合のerror
type UnexpectedInstanceStatusError struct {
	// Err エラー詳細
	Err error
}

// Error errorインターフェース実装
func (e *UnexpectedInstanceStatusError) Error() string {
	return fmt.Sprintf("resource returns unexpected instance status value: %s", e.Err.Error())
}

// StatePollingWaiter ポーリングによりリソースの状態が変わるまで待機する
type StatePollingWaiter struct {
	// NotFoundRetry Readで404が返ってきた場合のリトライ回数
	//
	// アプライアンスなどの一部のリソースでは作成~起動完了までの間に404を返すことがある。
	// これに対応するためこのフィールドにて404発生の許容回数を指定可能にする。
	NotFoundRetry int

	// ReadFunc 対象リソースの状態を取得するためのfunc
	//
	// TargetAvailabilityを指定する場合はAvailabilityHolderを返す必要がある
	// もしAvailabilityHolderを実装しておらず、かつStateCheckFuncも未指定だった場合はタイムアウトまで完了しないため注意
	ReadFunc StateReadFunc

	// TargetAvailability 対象リソースのAvailabilityがこの状態になった場合になるまで待つ
	//
	// この値を指定する場合、ReadFuncにてAvailabilityHolderを返す必要がある。
	// AvailabilityがTargetAvailabilityとPendingAvailabilityで指定されていない状態になった場合はUnexpectedAvailabilityErrorを返す
	//
	// TargetAvailability(Pending)とTargetInstanceState(Pending)の両方が指定された場合は両方を満たすまで待つ
	// StateCheckFuncとの併用は不可。併用した場合はpanicする。
	TargetAvailability []types.EAvailability

	// PendingAvailability 対象リソースのAvailabilityがこの状態になった場合は待ちを継続する。
	//
	// 詳細はTargetAvailabilityのコメントを参照
	PendingAvailability []types.EAvailability

	// TargetInstanceStatus 対象リソースのInstanceStatusがこの状態になった場合になるまで待つ
	//
	// この値を指定する場合、ReadFuncにてInstanceStatusHolderを返す必要がある。
	// InstanceStatusがTargetInstanceStatusとPendinngInstanceStatusで指定されていない状態になった場合はUnexpectedInstanceStatusErrorを返す
	//
	// TargetAvailabilityとTargetInstanceStateの両方が指定された場合は両方を満たすまで待つ
	//
	// StateCheckFuncとの併用は不可。併用した場合はpanicする。
	TargetInstanceStatus []types.EServerInstanceStatus

	// PendingInstanceStatus 対象リソースのInstanceStatusがこの状態になった場合は待ちを継続する。
	//
	// 詳細はTargetInstanceStatusのコメントを参照
	PendingInstanceStatus []types.EServerInstanceStatus

	// StateCheckFunc ReadFuncで得たリソースの情報を元に待ちを継続するかの判定を行うためのfunc
	//
	// TargetAvailabilityとTargetInstanceStateとの併用は不可。併用した場合panicする
	StateCheckFunc StateCheckFunc

	// Timeout タイムアウト
	Timeout time.Duration // タイムアウト
	// PollingInterval ポーリング間隔
	PollingInterval time.Duration
}

func (w *StatePollingWaiter) validateFields() {
	if w.ReadFunc == nil {
		panic(errors.New("StatePollingWaiter has invalid setting: ReadFunc is required"))
	}

	if w.StateCheckFunc != nil && (len(w.TargetAvailability) > 0 || len(w.TargetInstanceStatus) > 0) {
		panic(errors.New("StatePollingWaiter has invalid setting: StateCheckFunc and TargetAvailability/TargetInstanceStatus can not use together"))
	}

	if w.StateCheckFunc == nil && len(w.TargetAvailability) == 0 && len(w.TargetInstanceStatus) == 0 {
		panic(errors.New("StatePollingWaiter has invalid setting: TargetAvailability or TargetInstanceState must have least 1 items when StateCheckFunc is not set"))
	}
}

func (w *StatePollingWaiter) defaults() {

	if w.Timeout == time.Duration(0) {
		w.Timeout = DefaultStatePollingTimeout
	}
	if w.PollingInterval == time.Duration(0) {
		w.PollingInterval = DefaultStatePollingInterval
	}
}

// WaitForState リソースが指定の状態になるまで待つ
func (w *StatePollingWaiter) WaitForState(ctx context.Context) (interface{}, error) {
	c, p, e := w.AsyncWaitForState(ctx)
	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case lastState := <-c:
			return lastState, nil
		case <-p:
			// noop
		case err := <-e:
			return nil, err
		}
	}
}

// AsyncWaitForState リソースが指定の状態になるまで待つ
func (w *StatePollingWaiter) AsyncWaitForState(ctx context.Context) (compCh <-chan interface{}, progressCh <-chan interface{}, errorCh <-chan error) {

	w.validateFields()
	w.defaults()

	compChan := make(chan interface{})
	progChan := make(chan interface{})
	errChan := make(chan error)

	ticker := time.NewTicker(w.PollingInterval)

	go func() {
		ctx, cancel := context.WithTimeout(ctx, w.Timeout)
		defer cancel()

		defer ticker.Stop()

		defer close(compChan)
		defer close(progChan)
		defer close(errChan)

		notFoundCounter := w.NotFoundRetry
		for {
			select {
			case <-ctx.Done():
				errChan <- ctx.Err()
				return
			case <-ticker.C:
				state, err := w.ReadFunc()

				if err != nil {
					if IsNotFoundError(err) {
						notFoundCounter--
						if notFoundCounter >= 0 {
							continue
						}
					}
					errChan <- err
					return
				}

				exit, err := w.handleState(state)
				if exit {
					compChan <- state
					return
				}

				if err != nil {
					errChan <- err
					return
				}

				if state != nil {
					progChan <- state
				}
			}
		}
	}()

	compCh = compChan
	progressCh = progChan
	errorCh = errChan
	return compCh, progressCh, errorCh
}

func (w *StatePollingWaiter) handleState(state interface{}) (bool, error) {
	if w.StateCheckFunc != nil {
		return w.StateCheckFunc(state)
	}

	availabilityHolder, hasAvailability := state.(accessor.Availability)
	instanceStateHolder, hasInstanceState := state.(accessor.InstanceStatus)

	switch {
	case hasAvailability && hasInstanceState:

		res1, err := w.handleAvailability(availabilityHolder)
		if err != nil {
			return false, err
		}
		res2, err := w.handleInstanceState(instanceStateHolder)
		if err != nil {
			return false, err
		}
		return res1 && res2, nil

	case hasAvailability:
		return w.handleAvailability(availabilityHolder)
	case hasInstanceState:
		return w.handleInstanceState(instanceStateHolder)
	default:
		// どちらのインターフェースも実装していない場合、stateが存在するだけでtrueとする
		return true, nil
	}
}

func (w *StatePollingWaiter) handleAvailability(state accessor.Availability) (bool, error) {
	if len(w.TargetAvailability) == 0 {
		return true, nil
	}
	v := state.GetAvailability()
	switch {
	case w.isInAvailability(v, w.TargetAvailability):
		return true, nil
	case w.isInAvailability(v, w.PendingAvailability):
		return false, nil
	default:
		return false, fmt.Errorf("got unexpected value of Availability: got %q", v)
	}
}

func (w *StatePollingWaiter) handleInstanceState(state accessor.InstanceStatus) (bool, error) {
	if len(w.TargetInstanceStatus) == 0 {
		return true, nil
	}
	v := state.GetInstanceStatus()
	switch {
	case w.isInInstanceStatus(v, w.TargetInstanceStatus):
		return true, nil
	case w.isInInstanceStatus(v, w.PendingInstanceStatus):
		return false, nil
	default:
		return false, fmt.Errorf("got unexpected value of InstanceState: got %q", v)
	}
}

func (w *StatePollingWaiter) isInAvailability(v types.EAvailability, conds []types.EAvailability) bool {
	for _, cond := range conds {
		if v == cond {
			return true
		}
	}
	return false
}

func (w *StatePollingWaiter) isInInstanceStatus(v types.EServerInstanceStatus, conds []types.EServerInstanceStatus) bool {
	for _, cond := range conds {
		if v == cond {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package jobs

import (
	"context"
	"fmt"
	"sync"
	"time"

	"strings"

	"strconv"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/crawler"
	"github.com/meifamily/ptt-alertor/models"
	"github.com/meifamily/ptt-alertor/models/article"
	"github.com/meifamily/ptt-alertor/models/pushsum"
	"github.com/meifamily/ptt-alertor/models/subscription"
	"github.com/meifamily/ptt-alertor/models/user"
)

// change overdueHour must change cronjob replacepushsumkey in the mean time
const overdueHour = 48 * time.Hour
const pauseCheckPushSum = 5 * time.Minute

var psCker *pushSumChecker
var pscOnce sync.Once

type pushSumChecker struct {
	Checker
	ch chan pushSumChecker
}

func NewPushSumChecker() *pushSumChecker {
	pscOnce.Do(func() {
		psCker = &pushSumChecker{}
		psCker.done = make(chan struct{})
		psCker.ch = make(chan pushSumChecker)
	})
	return psCker
}

func (psc pushSumChecker) String() string {
	textMap := map[string]string{
		"pushup":   "推文數",
		"pushdown": "噓文數",
	}
	subType := textMap[psc.subType]
	return fmt.Sprintf("%s@%s\r\n看板：%s；%s：%s%s", psc.word, psc.board, psc.board, subType, psc.word, psc.articles.StringWithPushSum())
}

type BoardArticles struct {
	board    string
	articles article.Articles
}

func (psc pushSumChecker) Stop() {
	psc.done <- struct{}{}
	log.Info("Pushsum Checker Stop")
}

func (psc pushSumChecker) Run() {
	baCh := make(chan BoardArticles)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			default:
				boards := pushsum.List()
				for _, board := range boards {
					ba := BoardArticles{board: board}
					psc.crawlArticles(ba, baCh)
				}
				time.Sleep(pauseCheckPushSum)
			}
		}
	}()

	for {
		select {
		case ba := <-baCh:
			psc.board = ba.board
			if len(ba.articles) > 0 {
				go psc.checkSubscribers(ba)
			}
		case pscker := <-psc.ch:
			ckCh <- pscker
		case <-psc.done:
			cancel()
			for len(baCh) > 0 {
				<-baCh
			}
			for len(psc.ch) > 0 {
				<-psc.ch
			}
			return
		}
	}
}

func (psc pushSumChecker) crawlArticles(ba BoardArticles, baCh chan BoardArticles) {
	currentPage, err := crawler.CurrentPage(ba.board)
	if err != nil {
		log.WithFields(log.Fields{
			"board": ba.board,
		}).WithError(err).Error("Get CurrentPage Failed")
		baCh <- ba
		return
	}

Page:
	for page := currentPage; page > 0; page-- {
		articles, _ := crawler.BuildArticles(ba.board, page)
		for i := len(articles) - 1; i > 0; i-- {
			a := articles[i]
			if a.ID == 0 {
				continue
			}
			loc := time.FixedZone("CST", 8*60*60)
			t, err := time.ParseInLocation("1/02", a.Date, loc)
			now := time.Now()
			nowDate := now.Truncate(24 * time.Hour)
			if t.Month() > now.Month() {
				t = t.AddDate(now.Year()-1, 0, 0)
			} else {
				t = t.AddDate(now.Year(), 0, 0)
			}
			if err != nil {
				log.WithFields(log.Fields{
					"board": ba.board,
					"page":  page,
				}).WithError(err).Error("Parse DateTime Error")
				continue
			}
			if nowDate.After(t.Add(overdueHour)) {
				break Page
			}
			ba.articles = append(ba.articles, a)
		}
	}

	log.WithFields(log.Fields{
		"board": ba.board,
		"total": len(ba.articles),
	}).Info("PushSum Crawl Finish")

	baCh <- ba
}

func (psc pushSumChecker) checkSubscribers(ba BoardArticles) {
	subs := pushsum.ListSubscribers(ba.board)
	for _, account := range subs {
		u := models.User.Find(account)
		psc.Profile = u.Profile
		go psc.checkPushSum(u, ba, checkUp)
		go psc.checkPushSum(u, ba, checkDown)
	}
}

type checkPushSumFn func(*pushSumChecker, subscription.Subscription, article.Articles) (article.Articles, []int)

func checkUp(psc *pushSumChecker, sub subscription.Subscription, articles article.Articles) (upArticles article.Articles, ids []int) {
	psc.word = strconv.Itoa(sub.Up)
	psc.subType = "pushup"
	if sub.Up != 0 {
		for _, a := range articles {
			if a.PushSum >= sub.Up {
				upArticles = append(upArticles, a)
				ids = append(ids, a.ID)
			}
		}
	}
	return upArticles, ids
}

func checkDown(psc *pushSumChecker, sub subscription.Subscription, articles article.Articles) (downArticles article.Articles, ids []int) {
	down := sub.Down * -1
	psc.word = strconv.Itoa(down)
	psc.subType = "pushdown"
	if sub.Down != 0 {
		for _, a := range articles {
			if a.PushSum <= down {
				downArticles = append(downArticles, a)
				ids = append(ids, a.ID)
			}
		}
	}
	return downArticles, ids
}

func (psc pushSumChecker) checkPushSum(u user.User, ba BoardArticles, checkFn checkPushSumFn) {
	var articles article.Articles
	var ids []int
	for _, sub := range u.Subscribes {
		if strings.EqualFold(sub.Board, ba.board) {
			articles, ids = checkFn(&psc, sub, ba.articles)
		}
	}
	if len(articles) > 0 {
		psc.articles = psc.toSendArticles(ids, articles)
		if len(psc.articles) > 0 {
			psc.ch <- psc
		}
	}
}

func (psc pushSumChecker) toSendArticles(ids []int, articles article.Articles) article.Articles {
	kindMap := map[string]string{
		"pushup":   "up",
		"pushdown": "down",
	}
	ids = pushsum.DiffList(psc.Profile.Account, psc.board, kindMap[psc.subType], ids...)
	diffIds := make(map[int]bool)
	for _, id := range ids {
		diffIds[id] = true
	}
	sendArticles := make(article.Articles, 0)
	for _, a := range articles {
		if diffIds[a.ID] {
			sendArticles = append(sendArticles, a)
		}
	}
	return sendArticles
}
<commit_msg>:zap: crawl pushsum page parallel<commit_after>package jobs

import (
	"context"
	"fmt"
	"sync"
	"time"

	"strings"

	"strconv"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/crawler"
	"github.com/meifamily/ptt-alertor/models"
	"github.com/meifamily/ptt-alertor/models/article"
	"github.com/meifamily/ptt-alertor/models/pushsum"
	"github.com/meifamily/ptt-alertor/models/subscription"
	"github.com/meifamily/ptt-alertor/models/user"
)

// NewPushSumKeyReplacer Job schedule must longer than overduehour
const overdueHour = 48 * time.Hour

var psCker *pushSumChecker
var pscOnce sync.Once

type pushSumChecker struct {
	Checker
	ch       chan pushSumChecker
	duration time.Duration
}

func NewPushSumChecker() *pushSumChecker {
	pscOnce.Do(func() {
		psCker = &pushSumChecker{
			duration: 3 * time.Second,
		}
		psCker.done = make(chan struct{})
		psCker.ch = make(chan pushSumChecker)
	})
	return psCker
}

func (psc pushSumChecker) String() string {
	textMap := map[string]string{
		"pushup":   "推文數",
		"pushdown": "噓文數",
	}
	subType := textMap[psc.subType]
	return fmt.Sprintf("%s@%s\r\n看板：%s；%s：%s%s", psc.word, psc.board, psc.board, subType, psc.word, psc.articles.StringWithPushSum())
}

type BoardArticles struct {
	board    string
	articles article.Articles
}

func (psc pushSumChecker) Stop() {
	psc.done <- struct{}{}
	log.Info("Pushsum Checker Stop")
}

func (psc pushSumChecker) Run() {
	baCh := make(chan BoardArticles)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			default:
				boards := pushsum.List()
				for _, board := range boards {
					ba := BoardArticles{board: board}
					time.Sleep(psc.duration)
					go psc.crawlArticles(ba, baCh)
				}
			}
		}
	}()

	for {
		select {
		case ba := <-baCh:
			psc.board = ba.board
			if len(ba.articles) > 0 {
				go psc.checkSubscribers(ba)
			}
		case pscker := <-psc.ch:
			ckCh <- pscker
		case <-psc.done:
			cancel()
			for len(baCh) > 0 {
				<-baCh
			}
			for len(psc.ch) > 0 {
				<-psc.ch
			}
			return
		}
	}
}

func (psc pushSumChecker) crawlArticles(ba BoardArticles, baCh chan BoardArticles) {
	currentPage, err := crawler.CurrentPage(ba.board)
	if err != nil {
		log.WithFields(log.Fields{
			"board": ba.board,
		}).WithError(err).Error("Get CurrentPage Failed")
		baCh <- ba
		return
	}

Page:
	for page := currentPage; page > 0; page-- {
		articles, _ := crawler.BuildArticles(ba.board, page)
		for i := len(articles) - 1; i > 0; i-- {
			a := articles[i]
			if a.ID == 0 {
				continue
			}
			loc := time.FixedZone("CST", 8*60*60)
			t, err := time.ParseInLocation("1/02", a.Date, loc)
			now := time.Now()
			nowDate := now.Truncate(24 * time.Hour)
			if t.Month() > now.Month() {
				t = t.AddDate(now.Year()-1, 0, 0)
			} else {
				t = t.AddDate(now.Year(), 0, 0)
			}
			if err != nil {
				log.WithFields(log.Fields{
					"board": ba.board,
					"page":  page,
				}).WithError(err).Error("Parse DateTime Error")
				continue
			}
			if nowDate.After(t.Add(overdueHour)) {
				break Page
			}
			ba.articles = append(ba.articles, a)
		}
	}

	log.WithFields(log.Fields{
		"board": ba.board,
		"total": len(ba.articles),
	}).Info("PushSum Crawl Finish")

	baCh <- ba
}

func (psc pushSumChecker) checkSubscribers(ba BoardArticles) {
	subs := pushsum.ListSubscribers(ba.board)
	for _, account := range subs {
		u := models.User.Find(account)
		psc.Profile = u.Profile
		go psc.checkPushSum(u, ba, checkUp)
		go psc.checkPushSum(u, ba, checkDown)
	}
}

type checkPushSumFn func(*pushSumChecker, subscription.Subscription, article.Articles) (article.Articles, []int)

func checkUp(psc *pushSumChecker, sub subscription.Subscription, articles article.Articles) (upArticles article.Articles, ids []int) {
	psc.word = strconv.Itoa(sub.Up)
	psc.subType = "pushup"
	if sub.Up != 0 {
		for _, a := range articles {
			if a.PushSum >= sub.Up {
				upArticles = append(upArticles, a)
				ids = append(ids, a.ID)
			}
		}
	}
	return upArticles, ids
}

func checkDown(psc *pushSumChecker, sub subscription.Subscription, articles article.Articles) (downArticles article.Articles, ids []int) {
	down := sub.Down * -1
	psc.word = strconv.Itoa(down)
	psc.subType = "pushdown"
	if sub.Down != 0 {
		for _, a := range articles {
			if a.PushSum <= down {
				downArticles = append(downArticles, a)
				ids = append(ids, a.ID)
			}
		}
	}
	return downArticles, ids
}

func (psc pushSumChecker) checkPushSum(u user.User, ba BoardArticles, checkFn checkPushSumFn) {
	var articles article.Articles
	var ids []int
	for _, sub := range u.Subscribes {
		if strings.EqualFold(sub.Board, ba.board) {
			articles, ids = checkFn(&psc, sub, ba.articles)
		}
	}
	if len(articles) > 0 {
		psc.articles = psc.toSendArticles(ids, articles)
		if len(psc.articles) > 0 {
			psc.ch <- psc
		}
	}
}

func (psc pushSumChecker) toSendArticles(ids []int, articles article.Articles) article.Articles {
	kindMap := map[string]string{
		"pushup":   "up",
		"pushdown": "down",
	}
	ids = pushsum.DiffList(psc.Profile.Account, psc.board, kindMap[psc.subType], ids...)
	diffIds := make(map[int]bool)
	for _, id := range ids {
		diffIds[id] = true
	}
	sendArticles := make(article.Articles, 0)
	for _, a := range articles {
		if diffIds[a.ID] {
			sendArticles = append(sendArticles, a)
		}
	}
	return sendArticles
}
<|endoftext|>


<commit_before>package iris_test

import (
	"strconv"
	"testing"

	"gopkg.in/kataras/iris.v6"
	"gopkg.in/kataras/iris.v6/adaptors/gorillamux"
	"gopkg.in/kataras/iris.v6/httptest"
)

func newGorillaMuxAPP() *iris.Framework {
	app := iris.New()
	app.Adapt(gorillamux.New())

	return app
}

func TestGorillaMuxSimple(t *testing.T) {
	app := newGorillaMuxAPP()

	testRoutes := []testRoute{
		// FOUND - registered
		{"GET", "/test_get", "/test_get", "", "hello, get!", 200, true, nil, nil},
		{"POST", "/test_post", "/test_post", "", "hello, post!", 200, true, nil, nil},
		{"PUT", "/test_put", "/test_put", "", "hello, put!", 200, true, nil, nil},
		{"DELETE", "/test_delete", "/test_delete", "", "hello, delete!", 200, true, nil, nil},
		{"HEAD", "/test_head", "/test_head", "", "hello, head!", 200, true, nil, nil},
		{"OPTIONS", "/test_options", "/test_options", "", "hello, options!", 200, true, nil, nil},
		{"CONNECT", "/test_connect", "/test_connect", "", "hello, connect!", 200, true, nil, nil},
		{"PATCH", "/test_patch", "/test_patch", "", "hello, patch!", 200, true, nil, nil},
		{"TRACE", "/test_trace", "/test_trace", "", "hello, trace!", 200, true, nil, nil},
		// NOT FOUND - not registered
		{"GET", "/test_get_nofound", "/test_get_nofound", "", "Not Found", 404, false, nil, nil},
		{"POST", "/test_post_nofound", "/test_post_nofound", "", "Not Found", 404, false, nil, nil},
		{"PUT", "/test_put_nofound", "/test_put_nofound", "", "Not Found", 404, false, nil, nil},
		{"DELETE", "/test_delete_nofound", "/test_delete_nofound", "", "Not Found", 404, false, nil, nil},
		{"HEAD", "/test_head_nofound", "/test_head_nofound", "", "Not Found", 404, false, nil, nil},
		{"OPTIONS", "/test_options_nofound", "/test_options_nofound", "", "Not Found", 404, false, nil, nil},
		{"CONNECT", "/test_connect_nofound", "/test_connect_nofound", "", "Not Found", 404, false, nil, nil},
		{"PATCH", "/test_patch_nofound", "/test_patch_nofound", "", "Not Found", 404, false, nil, nil},
		{"TRACE", "/test_trace_nofound", "/test_trace_nofound", "", "Not Found", 404, false, nil, nil},
		// Parameters
		{"GET", "/test_get_parameter1/{name}", "/test_get_parameter1/iris", "", "name=iris", 200, true, []param{{"name", "iris"}}, nil},
		{"GET", "/test_get_parameter2/{name}/details/{something}", "/test_get_parameter2/iris/details/anything", "", "name=iris,something=anything", 200, true, []param{{"name", "iris"}, {"something", "anything"}}, nil},
		{"GET", "/test_get_parameter2/{name}/details/{something}/{else:.*}", "/test_get_parameter2/iris/details/anything/elsehere", "", "name=iris,something=anything,else=elsehere", 200, true, []param{{"name", "iris"}, {"something", "anything"}, {"else", "elsehere"}}, nil},
		// URL Parameters
		{"GET", "/test_get_urlparameter1/first", "/test_get_urlparameter1/first", "name=irisurl", "name=irisurl", 200, true, nil, []param{{"name", "irisurl"}}},
		{"GET", "/test_get_urlparameter2/second", "/test_get_urlparameter2/second", "name=irisurl&something=anything", "name=irisurl,something=anything", 200, true, nil, []param{{"name", "irisurl"}, {"something", "anything"}}},
		{"GET", "/test_get_urlparameter2/first/second/third", "/test_get_urlparameter2/first/second/third", "name=irisurl&something=anything&else=elsehere", "name=irisurl,something=anything,else=elsehere", 200, true, nil, []param{{"name", "irisurl"}, {"something", "anything"}, {"else", "elsehere"}}},
	}

	for idx := range testRoutes {
		r := testRoutes[idx]
		if r.Register {
			app.HandleFunc(r.Method, r.Path, func(ctx *iris.Context) {
				ctx.SetStatusCode(r.Status)
				if r.Params != nil && len(r.Params) > 0 {
					ctx.Writef(ctx.ParamsSentence())
				} else if r.URLParams != nil && len(r.URLParams) > 0 {
					if len(r.URLParams) != len(ctx.URLParams()) {
						t.Fatalf("Error when comparing length of url parameters %d != %d", len(r.URLParams), len(ctx.URLParams()))
					}
					paramsKeyVal := ""
					///TODO:
					// Gorilla mux saves and gets its vars by map, so no specific order
					//
					// I should change this test below:
					for idxp, p := range r.URLParams {
						val := ctx.URLParam(p.Key)
						paramsKeyVal += p.Key + "=" + val + ","
						if idxp == len(r.URLParams)-1 {
							paramsKeyVal = paramsKeyVal[0 : len(paramsKeyVal)-1]
						}
					}
					ctx.Writef(paramsKeyVal)
				} else {
					ctx.Writef(r.Body)
				}

			})
		}
	}

	e := httptest.New(app, t)

	// run the tests (1)
	for idx := range testRoutes {
		r := testRoutes[idx]
		e.Request(r.Method, r.RequestPath).WithQueryString(r.RequestQuery).
			Expect().
			Status(r.Status).Body().Equal(r.Body)
	}

}

func TestGorillaMuxSimpleParty(t *testing.T) {
	app := newGorillaMuxAPP()

	h := func(ctx *iris.Context) { ctx.WriteString(ctx.Host() + ctx.Path()) }

	if testEnableSubdomain {
		subdomainParty := app.Party(testSubdomain + ".")
		{
			subdomainParty.Get("/", h)
			subdomainParty.Get("/path1", h)
			subdomainParty.Get("/path2", h)
			subdomainParty.Get("/namedpath/{param1}/something/{param2}", h)
			subdomainParty.Get("/namedpath/{param1}/something/{param2}/else", h)
		}
	}

	// simple
	p := app.Party("/party1")
	{
		p.Get("/", h)
		p.Get("/path1", h)
		p.Get("/path2", h)
		p.Get("/namedpath/{param1}/something/{param2}", h)
		p.Get("/namedpath/{param1}/something/{param2}/else", h)
	}

	app.Config.VHost = "0.0.0.0:" + strconv.Itoa(getRandomNumber(2222, 2399))
	// app.Config.Tester.Debug = true
	// app.Config.Tester.ExplicitURL = true
	e := httptest.New(app, t)

	request := func(reqPath string) {

		e.Request("GET", reqPath).
			Expect().
			Status(iris.StatusOK).Body().Equal(app.Config.VHost + reqPath)
	}

	// run the tests
	request("/party1/")
	request("/party1/path1")
	request("/party1/path2")
	request("/party1/namedpath/theparam1/something/theparam2")
	request("/party1/namedpath/theparam1/something/theparam2/else")

	if testEnableSubdomain {
		es := subdomainTester(e, app)
		subdomainRequest := func(reqPath string) {
			es.Request("GET", reqPath).
				Expect().
				Status(iris.StatusOK).Body().Equal(testSubdomainHost(app.Config.VHost) + reqPath)
		}

		subdomainRequest("/")
		subdomainRequest("/path1")
		subdomainRequest("/path2")
		subdomainRequest("/namedpath/theparam1/something/theparam2")
		subdomainRequest("/namedpath/theparam1/something/theparam2/else")
	}
}

func TestGorillaMuxPathEscape(t *testing.T) {
	app := newGorillaMuxAPP()

	app.Get("/details/{name}", func(ctx *iris.Context) {
		name := ctx.Param("name")
		highlight := ctx.URLParam("highlight")
		ctx.Writef("name=%s,highlight=%s", name, highlight)
	})

	e := httptest.New(app, t)

	e.GET("/details/Sakamoto desu ga").
		WithQuery("highlight", "text").
		Expect().Status(iris.StatusOK).Body().Equal("name=Sakamoto desu ga,highlight=text")
}

func TestGorillaMuxParamDecodedDecodeURL(t *testing.T) {
	app := newGorillaMuxAPP()

	app.Get("/encoding/{url}", func(ctx *iris.Context) {
		url := iris.DecodeURL(ctx.ParamDecoded("url"))
		ctx.SetStatusCode(iris.StatusOK)
		ctx.WriteString(url)
	})

	e := httptest.New(app, t)

	e.GET("/encoding/http%3A%2F%2Fsome-url.com").Expect().Status(iris.StatusOK).Body().Equal("http://some-url.com")
}

func TestGorillaMuxRouteURLPath(t *testing.T) {
	app := iris.New()
	app.Adapt(gorillamux.New())

	app.None("/profile/{user_id}/{ref}/{anything:.*}", nil).ChangeName("profile")
	app.Boot()

	expected := "/profile/42/iris-go/something"

	if got := app.Path("profile", "user_id", 42, "ref", "iris-go", "anything", "something"); got != expected {
		t.Fatalf("gorillamux' reverse routing 'URLPath' error:  expected %s but got %s", expected, got)
	}
}
<commit_msg>test  gorillamux params- order doesn't matters, todo done.<commit_after>package iris_test

import (
	"strconv"
	"testing"

	"gopkg.in/kataras/iris.v6"
	"gopkg.in/kataras/iris.v6/adaptors/gorillamux"
	"gopkg.in/kataras/iris.v6/httptest"
)

func newGorillaMuxAPP() *iris.Framework {
	app := iris.New()
	app.Adapt(gorillamux.New())

	return app
}

func TestGorillaMuxSimple(t *testing.T) {
	app := newGorillaMuxAPP()

	testRoutes := []testRoute{
		// FOUND - registered
		{"GET", "/test_get", "/test_get", "", "hello, get!", 200, true, nil, nil},
		{"POST", "/test_post", "/test_post", "", "hello, post!", 200, true, nil, nil},
		{"PUT", "/test_put", "/test_put", "", "hello, put!", 200, true, nil, nil},
		{"DELETE", "/test_delete", "/test_delete", "", "hello, delete!", 200, true, nil, nil},
		{"HEAD", "/test_head", "/test_head", "", "hello, head!", 200, true, nil, nil},
		{"OPTIONS", "/test_options", "/test_options", "", "hello, options!", 200, true, nil, nil},
		{"CONNECT", "/test_connect", "/test_connect", "", "hello, connect!", 200, true, nil, nil},
		{"PATCH", "/test_patch", "/test_patch", "", "hello, patch!", 200, true, nil, nil},
		{"TRACE", "/test_trace", "/test_trace", "", "hello, trace!", 200, true, nil, nil},
		// NOT FOUND - not registered
		{"GET", "/test_get_nofound", "/test_get_nofound", "", "Not Found", 404, false, nil, nil},
		{"POST", "/test_post_nofound", "/test_post_nofound", "", "Not Found", 404, false, nil, nil},
		{"PUT", "/test_put_nofound", "/test_put_nofound", "", "Not Found", 404, false, nil, nil},
		{"DELETE", "/test_delete_nofound", "/test_delete_nofound", "", "Not Found", 404, false, nil, nil},
		{"HEAD", "/test_head_nofound", "/test_head_nofound", "", "Not Found", 404, false, nil, nil},
		{"OPTIONS", "/test_options_nofound", "/test_options_nofound", "", "Not Found", 404, false, nil, nil},
		{"CONNECT", "/test_connect_nofound", "/test_connect_nofound", "", "Not Found", 404, false, nil, nil},
		{"PATCH", "/test_patch_nofound", "/test_patch_nofound", "", "Not Found", 404, false, nil, nil},
		{"TRACE", "/test_trace_nofound", "/test_trace_nofound", "", "Not Found", 404, false, nil, nil},
		// Parameters
		{"GET", "/test_get_parameter1/{name}", "/test_get_parameter1/iris", "", "name=iris", 200, true, []param{{"name", "iris"}}, nil},
		{"GET", "/test_get_parameter2/{name}/details/{something}", "/test_get_parameter2/iris/details/anything", "", "name=iris,something=anything", 200, true, []param{{"name", "iris"}, {"something", "anything"}}, nil},
		{"GET", "/test_get_parameter2/{name}/details/{something}/{else:.*}", "/test_get_parameter2/iris/details/anything/elsehere", "", "name=iris,something=anything,else=elsehere", 200, true, []param{{"name", "iris"}, {"something", "anything"}, {"else", "elsehere"}}, nil},
		// URL Parameters
		{"GET", "/test_get_urlparameter1/first", "/test_get_urlparameter1/first", "name=irisurl", "name=irisurl", 200, true, nil, []param{{"name", "irisurl"}}},
		{"GET", "/test_get_urlparameter2/second", "/test_get_urlparameter2/second", "name=irisurl&something=anything", "name=irisurl,something=anything", 200, true, nil, []param{{"name", "irisurl"}, {"something", "anything"}}},
		{"GET", "/test_get_urlparameter2/first/second/third", "/test_get_urlparameter2/first/second/third", "name=irisurl&something=anything&else=elsehere", "name=irisurl,something=anything,else=elsehere", 200, true, nil, []param{{"name", "irisurl"}, {"something", "anything"}, {"else", "elsehere"}}},
	}

	for idx := range testRoutes {
		r := testRoutes[idx]
		if r.Register {
			app.HandleFunc(r.Method, r.Path, func(ctx *iris.Context) {
				ctx.SetStatusCode(r.Status)
				if r.Params != nil && len(r.Params) > 0 {
					ctx.Writef(ctx.ParamsSentence())
				} else if r.URLParams != nil && len(r.URLParams) > 0 {
					if len(r.URLParams) != len(ctx.URLParams()) {
						t.Fatalf("Error when comparing length of url parameters %d != %d", len(r.URLParams), len(ctx.URLParams()))
					}
					paramsKeyVal := ""

					for idxp, p := range r.URLParams {
						val := ctx.URLParam(p.Key)
						paramsKeyVal += p.Key + "=" + val + ","
						if idxp == len(r.URLParams)-1 {
							paramsKeyVal = paramsKeyVal[0 : len(paramsKeyVal)-1]
						}
					}
					ctx.Writef(paramsKeyVal)
				} else {
					ctx.Writef(r.Body)
				}

			})
		}
	}

	e := httptest.New(app, t)

	// run the tests (1)
	for idx := range testRoutes {
		r := testRoutes[idx]
		e.Request(r.Method, r.RequestPath).WithQueryString(r.RequestQuery).
			Expect().
			// compare just the Len because gorillamux gets and sets the vars as map, so the values are unorderded.
			Status(r.Status).Body().Length().Equal(len(r.Body))
	}

}

func TestGorillaMuxSimpleParty(t *testing.T) {
	app := newGorillaMuxAPP()

	h := func(ctx *iris.Context) { ctx.WriteString(ctx.Host() + ctx.Path()) }

	if testEnableSubdomain {
		subdomainParty := app.Party(testSubdomain + ".")
		{
			subdomainParty.Get("/", h)
			subdomainParty.Get("/path1", h)
			subdomainParty.Get("/path2", h)
			subdomainParty.Get("/namedpath/{param1}/something/{param2}", h)
			subdomainParty.Get("/namedpath/{param1}/something/{param2}/else", h)
		}
	}

	// simple
	p := app.Party("/party1")
	{
		p.Get("/", h)
		p.Get("/path1", h)
		p.Get("/path2", h)
		p.Get("/namedpath/{param1}/something/{param2}", h)
		p.Get("/namedpath/{param1}/something/{param2}/else", h)
	}

	app.Config.VHost = "0.0.0.0:" + strconv.Itoa(getRandomNumber(2222, 2399))
	// app.Config.Tester.Debug = true
	// app.Config.Tester.ExplicitURL = true
	e := httptest.New(app, t)

	request := func(reqPath string) {

		e.Request("GET", reqPath).
			Expect().
			Status(iris.StatusOK).Body().Equal(app.Config.VHost + reqPath)
	}

	// run the tests
	request("/party1/")
	request("/party1/path1")
	request("/party1/path2")
	request("/party1/namedpath/theparam1/something/theparam2")
	request("/party1/namedpath/theparam1/something/theparam2/else")

	if testEnableSubdomain {
		es := subdomainTester(e, app)
		subdomainRequest := func(reqPath string) {
			es.Request("GET", reqPath).
				Expect().
				Status(iris.StatusOK).Body().Equal(testSubdomainHost(app.Config.VHost) + reqPath)
		}

		subdomainRequest("/")
		subdomainRequest("/path1")
		subdomainRequest("/path2")
		subdomainRequest("/namedpath/theparam1/something/theparam2")
		subdomainRequest("/namedpath/theparam1/something/theparam2/else")
	}
}

func TestGorillaMuxPathEscape(t *testing.T) {
	app := newGorillaMuxAPP()

	app.Get("/details/{name}", func(ctx *iris.Context) {
		name := ctx.Param("name")
		highlight := ctx.URLParam("highlight")
		ctx.Writef("name=%s,highlight=%s", name, highlight)
	})

	e := httptest.New(app, t)

	e.GET("/details/Sakamoto desu ga").
		WithQuery("highlight", "text").
		Expect().Status(iris.StatusOK).Body().Equal("name=Sakamoto desu ga,highlight=text")
}

func TestGorillaMuxParamDecodedDecodeURL(t *testing.T) {
	app := newGorillaMuxAPP()

	app.Get("/encoding/{url}", func(ctx *iris.Context) {
		url := iris.DecodeURL(ctx.ParamDecoded("url"))
		ctx.SetStatusCode(iris.StatusOK)
		ctx.WriteString(url)
	})

	e := httptest.New(app, t)

	e.GET("/encoding/http%3A%2F%2Fsome-url.com").Expect().Status(iris.StatusOK).Body().Equal("http://some-url.com")
}

func TestGorillaMuxRouteURLPath(t *testing.T) {
	app := iris.New()
	app.Adapt(gorillamux.New())

	app.None("/profile/{user_id}/{ref}/{anything:.*}", nil).ChangeName("profile")
	app.Boot()

	expected := "/profile/42/iris-go/something"

	if got := app.Path("profile", "user_id", 42, "ref", "iris-go", "anything", "something"); got != expected {
		t.Fatalf("gorillamux' reverse routing 'URLPath' error:  expected %s but got %s", expected, got)
	}
}
<|endoftext|>


<commit_before>package veneur

import (
	"math"
	"math/rand"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestCounterEmpty(t *testing.T) {

	c := NewCounter("a.b.c", []string{"a:b"})
	c.Sample(1, 1.0)

	assert.Equal(t, "a.b.c", c.name, "Name")
	assert.Len(t, c.tags, 1, "Tag length")
	assert.Equal(t, c.tags[0], "a:b", "Tag contents")

	metrics := c.Flush(10 * time.Second)
	assert.Len(t, metrics, 1, "Flushes 1 metric")

	m1 := metrics[0]
	assert.Equal(t, int32(10), m1.Interval, "Interval")
	assert.Equal(t, "rate", m1.MetricType, "Type")
	assert.Len(t, c.tags, 1, "Tag length")
	assert.Equal(t, c.tags[0], "a:b", "Tag contents")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, 0.1, m1.Value[0][1], "Metric value")
}

func TestCounterRate(t *testing.T) {

	c := NewCounter("a.b.c", []string{"a:b"})

	c.Sample(5, 1.0)

	// The counter returns an array with a single tuple of timestamp,value
	metrics := c.Flush(10 * time.Second)
	assert.Equal(t, 0.5, metrics[0].Value[0][1], "Metric value")
}

func TestCounterSampleRate(t *testing.T) {

	c := NewCounter("a.b.c", []string{"a:b"})

	c.Sample(5, 0.5)

	// The counter returns an array with a single tuple of timestamp,value
	metrics := c.Flush(10 * time.Second)
	assert.Equal(t, float64(1), metrics[0].Value[0][1], "Metric value")
}

func TestGauge(t *testing.T) {

	g := NewGauge("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", g.name, "Name")
	assert.Len(t, g.tags, 1, "Tag length")
	assert.Equal(t, g.tags[0], "a:b", "Tag contents")

	g.Sample(5, 1.0)

	metrics := g.Flush()
	assert.Len(t, metrics, 1, "Flushed metric count")

	m1 := metrics[0]
	// Interval is not meaningful for this
	assert.Equal(t, int32(0), m1.Interval, "Interval")
	assert.Equal(t, "gauge", m1.MetricType, "Type")
	tags := m1.Tags
	assert.Len(t, tags, 1, "Tag length")
	assert.Equal(t, tags[0], "a:b", "Tag contents")

	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(5), m1.Value[0][1], "Value")
}

func TestSet(t *testing.T) {
	s := NewSet("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", s.name, "Name")
	assert.Len(t, s.tags, 1, "Tag count")
	assert.Equal(t, "a:b", s.tags[0], "First tag")

	s.Sample("5", 1.0)

	s.Sample("5", 1.0)

	s.Sample("123", 1.0)

	s.Sample("2147483647", 1.0)
	s.Sample("-2147483648", 1.0)

	metrics := s.Flush()
	assert.Len(t, metrics, 1, "Flush")

	m1 := metrics[0]
	// Interval is not meaningful for this
	assert.Equal(t, int32(0), m1.Interval, "Interval")
	assert.Equal(t, "gauge", m1.MetricType, "Type")
	assert.Len(t, m1.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m1.Tags[0], "First tag")
	assert.Equal(t, float64(4), m1.Value[0][1], "Value")
}

func TestSetMerge(t *testing.T) {
	rand.Seed(time.Now().Unix())

	s := NewSet("a.b.c", []string{"a:b"})
	for i := 0; i < 100; i++ {
		s.Sample(strconv.Itoa(rand.Int()), 1.0)
	}
	assert.Equal(t, uint64(100), s.hll.Count(), "counts did not match")

	jm, err := s.Export()
	assert.NoError(t, err, "should have exported successfully")

	s2 := NewSet("a.b.c", []string{"a:b"})
	assert.NoError(t, s2.Combine(jm.Value), "should have combined successfully")
	assert.Equal(t, s.hll.Count(), s2.hll.Count(), "counts did not match after merging")
}

func TestHisto(t *testing.T) {

	h := NewHist("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", h.name, "Name")
	assert.Len(t, h.tags, 1, "Tag count")
	assert.Equal(t, "a:b", h.tags[0], "First tag")

	h.Sample(5, 1.0)
	h.Sample(10, 1.0)
	h.Sample(15, 1.0)
	h.Sample(20, 1.0)
	h.Sample(25, 1.0)

	metrics := h.Flush(10*time.Second, []float64{0.50})
	// We get lots of metrics back for histograms!
	assert.Len(t, metrics, 4, "Flushed metrics length")

	// the max
	m2 := metrics[0]
	assert.Equal(t, "a.b.c.max", m2.Name, "Name")
	assert.Equal(t, int32(0), m2.Interval, "Interval")
	assert.Equal(t, "gauge", m2.MetricType, "Type")
	assert.Len(t, m2.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m2.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(25), m2.Value[0][1], "Value")

	// the min
	m3 := metrics[1]
	assert.Equal(t, "a.b.c.min", m3.Name, "Name")
	assert.Equal(t, int32(0), m3.Interval, "Interval")
	assert.Equal(t, "gauge", m3.MetricType, "Type")
	assert.Len(t, m3.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m3.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(5), m3.Value[0][1], "Value")

	// the count
	m1 := metrics[2]
	assert.Equal(t, "a.b.c.count", m1.Name, "Name")
	assert.Equal(t, int32(10), m1.Interval, "Interval")
	assert.Equal(t, "rate", m1.MetricType, "Type")
	assert.Len(t, m1.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m1.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(0.5), m1.Value[0][1], "Value")

	// And the percentile
	m4 := metrics[3]
	assert.Equal(t, "a.b.c.50percentile", m4.Name, "Name")
	assert.Equal(t, int32(0), m4.Interval, "Interval")
	assert.Equal(t, "gauge", m4.MetricType, "Type")
	assert.Len(t, m4.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m4.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(15), m4.Value[0][1], "Value")
}

func TestHistoSampleRate(t *testing.T) {

	h := NewHist("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", h.name, "Name")
	assert.Len(t, h.tags, 1, "Tag length")
	assert.Equal(t, h.tags[0], "a:b", "Tag contents")

	h.Sample(5, 0.5)
	h.Sample(10, 0.5)
	h.Sample(15, 0.5)
	h.Sample(20, 0.5)
	h.Sample(25, 0.5)

	metrics := h.Flush(10*time.Second, []float64{0.50})
	assert.Len(t, metrics, 4, "Metrics flush length")

	// First the max
	m1 := metrics[0]
	assert.Equal(t, "a.b.c.max", m1.Name, "Max name")
	assert.Equal(t, float64(25), m1.Value[0][1], "Sampled max as rate")

	count := metrics[2]
	assert.Equal(t, "a.b.c.count", count.Name, "count name")
	assert.Equal(t, float64(1), count.Value[0][1], "count value")
}

func TestHistoMerge(t *testing.T) {
	rand.Seed(time.Now().Unix())

	h := NewHist("a.b.c", []string{"a:b"})
	for i := 0; i < 100; i++ {
		h.Sample(rand.NormFloat64(), 1.0)
	}

	jm, err := h.Export()
	assert.NoError(t, err, "should have exported successfully")

	h2 := NewHist("a.b.c", []string{"a:b"})
	assert.NoError(t, h2.Combine(jm.Value), "should have combined successfully")
	assert.InEpsilon(t, h.value.Quantile(0.5), h2.value.Quantile(0.5), 0.02, "50th percentiles did not match after merging")
	assert.InDelta(t, 0, h2.localWeight, 0.02, "merged histogram should have count of zero")
	assert.True(t, math.IsInf(h2.localMin, +1), "merged histogram should have local minimum of +inf")
	assert.True(t, math.IsInf(h2.localMax, -1), "merged histogram should have local minimum of -inf")

	h2.Sample(1.0, 1.0)
	assert.InDelta(t, 1.0, h2.localWeight, 0.02, "merged histogram should have count of 1 after adding a value")
	assert.InDelta(t, 1.0, h2.localMin, 0.02, "merged histogram should have min of 1 after adding a value")
	assert.InDelta(t, 1.0, h2.localMax, 0.02, "merged histogram should have max of 1 after adding a value")
}
<commit_msg>Reduce flakiness of hyperloglog test<commit_after>package veneur

import (
	"math"
	"math/rand"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestCounterEmpty(t *testing.T) {

	c := NewCounter("a.b.c", []string{"a:b"})
	c.Sample(1, 1.0)

	assert.Equal(t, "a.b.c", c.name, "Name")
	assert.Len(t, c.tags, 1, "Tag length")
	assert.Equal(t, c.tags[0], "a:b", "Tag contents")

	metrics := c.Flush(10 * time.Second)
	assert.Len(t, metrics, 1, "Flushes 1 metric")

	m1 := metrics[0]
	assert.Equal(t, int32(10), m1.Interval, "Interval")
	assert.Equal(t, "rate", m1.MetricType, "Type")
	assert.Len(t, c.tags, 1, "Tag length")
	assert.Equal(t, c.tags[0], "a:b", "Tag contents")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, 0.1, m1.Value[0][1], "Metric value")
}

func TestCounterRate(t *testing.T) {

	c := NewCounter("a.b.c", []string{"a:b"})

	c.Sample(5, 1.0)

	// The counter returns an array with a single tuple of timestamp,value
	metrics := c.Flush(10 * time.Second)
	assert.Equal(t, 0.5, metrics[0].Value[0][1], "Metric value")
}

func TestCounterSampleRate(t *testing.T) {

	c := NewCounter("a.b.c", []string{"a:b"})

	c.Sample(5, 0.5)

	// The counter returns an array with a single tuple of timestamp,value
	metrics := c.Flush(10 * time.Second)
	assert.Equal(t, float64(1), metrics[0].Value[0][1], "Metric value")
}

func TestGauge(t *testing.T) {

	g := NewGauge("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", g.name, "Name")
	assert.Len(t, g.tags, 1, "Tag length")
	assert.Equal(t, g.tags[0], "a:b", "Tag contents")

	g.Sample(5, 1.0)

	metrics := g.Flush()
	assert.Len(t, metrics, 1, "Flushed metric count")

	m1 := metrics[0]
	// Interval is not meaningful for this
	assert.Equal(t, int32(0), m1.Interval, "Interval")
	assert.Equal(t, "gauge", m1.MetricType, "Type")
	tags := m1.Tags
	assert.Len(t, tags, 1, "Tag length")
	assert.Equal(t, tags[0], "a:b", "Tag contents")

	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(5), m1.Value[0][1], "Value")
}

func TestSet(t *testing.T) {
	s := NewSet("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", s.name, "Name")
	assert.Len(t, s.tags, 1, "Tag count")
	assert.Equal(t, "a:b", s.tags[0], "First tag")

	s.Sample("5", 1.0)

	s.Sample("5", 1.0)

	s.Sample("123", 1.0)

	s.Sample("2147483647", 1.0)
	s.Sample("-2147483648", 1.0)

	metrics := s.Flush()
	assert.Len(t, metrics, 1, "Flush")

	m1 := metrics[0]
	// Interval is not meaningful for this
	assert.Equal(t, int32(0), m1.Interval, "Interval")
	assert.Equal(t, "gauge", m1.MetricType, "Type")
	assert.Len(t, m1.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m1.Tags[0], "First tag")
	assert.Equal(t, float64(4), m1.Value[0][1], "Value")
}

func TestSetMerge(t *testing.T) {
	rand.Seed(time.Now().Unix())

	s := NewSet("a.b.c", []string{"a:b"})
	for i := 0; i < 100; i++ {
		s.Sample(strconv.Itoa(rand.Int()), 1.0)
	}
	assert.Equal(t, uint64(100), s.hll.Count(), "counts did not match")

	jm, err := s.Export()
	assert.NoError(t, err, "should have exported successfully")

	s2 := NewSet("a.b.c", []string{"a:b"})
	assert.NoError(t, s2.Combine(jm.Value), "should have combined successfully")
	// HLLs are approximate, and we've seen error of +-1 here in the past, so
	// we're giving the test some room for error to reduce flakes
	countDifference := int(s.hll.Count()) - int(s2.hll.Count())
	assert.True(t, -1 < countDifference && countDifference < 1, "counts did not match after merging")
}

func TestHisto(t *testing.T) {

	h := NewHist("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", h.name, "Name")
	assert.Len(t, h.tags, 1, "Tag count")
	assert.Equal(t, "a:b", h.tags[0], "First tag")

	h.Sample(5, 1.0)
	h.Sample(10, 1.0)
	h.Sample(15, 1.0)
	h.Sample(20, 1.0)
	h.Sample(25, 1.0)

	metrics := h.Flush(10*time.Second, []float64{0.50})
	// We get lots of metrics back for histograms!
	assert.Len(t, metrics, 4, "Flushed metrics length")

	// the max
	m2 := metrics[0]
	assert.Equal(t, "a.b.c.max", m2.Name, "Name")
	assert.Equal(t, int32(0), m2.Interval, "Interval")
	assert.Equal(t, "gauge", m2.MetricType, "Type")
	assert.Len(t, m2.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m2.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(25), m2.Value[0][1], "Value")

	// the min
	m3 := metrics[1]
	assert.Equal(t, "a.b.c.min", m3.Name, "Name")
	assert.Equal(t, int32(0), m3.Interval, "Interval")
	assert.Equal(t, "gauge", m3.MetricType, "Type")
	assert.Len(t, m3.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m3.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(5), m3.Value[0][1], "Value")

	// the count
	m1 := metrics[2]
	assert.Equal(t, "a.b.c.count", m1.Name, "Name")
	assert.Equal(t, int32(10), m1.Interval, "Interval")
	assert.Equal(t, "rate", m1.MetricType, "Type")
	assert.Len(t, m1.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m1.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(0.5), m1.Value[0][1], "Value")

	// And the percentile
	m4 := metrics[3]
	assert.Equal(t, "a.b.c.50percentile", m4.Name, "Name")
	assert.Equal(t, int32(0), m4.Interval, "Interval")
	assert.Equal(t, "gauge", m4.MetricType, "Type")
	assert.Len(t, m4.Tags, 1, "Tag count")
	assert.Equal(t, "a:b", m4.Tags[0], "First tag")
	// The counter returns an array with a single tuple of timestamp,value
	assert.Equal(t, float64(15), m4.Value[0][1], "Value")
}

func TestHistoSampleRate(t *testing.T) {

	h := NewHist("a.b.c", []string{"a:b"})

	assert.Equal(t, "a.b.c", h.name, "Name")
	assert.Len(t, h.tags, 1, "Tag length")
	assert.Equal(t, h.tags[0], "a:b", "Tag contents")

	h.Sample(5, 0.5)
	h.Sample(10, 0.5)
	h.Sample(15, 0.5)
	h.Sample(20, 0.5)
	h.Sample(25, 0.5)

	metrics := h.Flush(10*time.Second, []float64{0.50})
	assert.Len(t, metrics, 4, "Metrics flush length")

	// First the max
	m1 := metrics[0]
	assert.Equal(t, "a.b.c.max", m1.Name, "Max name")
	assert.Equal(t, float64(25), m1.Value[0][1], "Sampled max as rate")

	count := metrics[2]
	assert.Equal(t, "a.b.c.count", count.Name, "count name")
	assert.Equal(t, float64(1), count.Value[0][1], "count value")
}

func TestHistoMerge(t *testing.T) {
	rand.Seed(time.Now().Unix())

	h := NewHist("a.b.c", []string{"a:b"})
	for i := 0; i < 100; i++ {
		h.Sample(rand.NormFloat64(), 1.0)
	}

	jm, err := h.Export()
	assert.NoError(t, err, "should have exported successfully")

	h2 := NewHist("a.b.c", []string{"a:b"})
	assert.NoError(t, h2.Combine(jm.Value), "should have combined successfully")
	assert.InEpsilon(t, h.value.Quantile(0.5), h2.value.Quantile(0.5), 0.02, "50th percentiles did not match after merging")
	assert.InDelta(t, 0, h2.localWeight, 0.02, "merged histogram should have count of zero")
	assert.True(t, math.IsInf(h2.localMin, +1), "merged histogram should have local minimum of +inf")
	assert.True(t, math.IsInf(h2.localMax, -1), "merged histogram should have local minimum of -inf")

	h2.Sample(1.0, 1.0)
	assert.InDelta(t, 1.0, h2.localWeight, 0.02, "merged histogram should have count of 1 after adding a value")
	assert.InDelta(t, 1.0, h2.localMin, 0.02, "merged histogram should have min of 1 after adding a value")
	assert.InDelta(t, 1.0, h2.localMax, 0.02, "merged histogram should have max of 1 after adding a value")
}
<|endoftext|>