<commit_before>package bridge

import (
	"fmt"
	"net"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/iptables"
	"github.com/docker/libnetwork/netutils"
)

// DockerChain: DOCKER iptable chain name
const (
	DockerChain    = "DOCKER"
	IsolationChain = "DOCKER-ISOLATION"
)

func setupIPChains(config *configuration) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
	// Sanity check.
	if config.EnableIPTables == false {
		return nil, nil, nil, fmt.Errorf("cannot create new chains, EnableIPTable is disabled")
	}

	hairpinMode := !config.EnableUserlandProxy

	natChain, err := iptables.NewChain(DockerChain, iptables.Nat, hairpinMode)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to create NAT chain: %v", err)
	}
	defer func() {
		if err != nil {
			if err := iptables.RemoveExistingChain(DockerChain, iptables.Nat); err != nil {
				logrus.Warnf("failed on removing iptables NAT chain on cleanup: %v", err)
			}
		}
	}()

	filterChain, err := iptables.NewChain(DockerChain, iptables.Filter, false)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to create FILTER chain: %v", err)
	}
	defer func() {
		if err != nil {
			if err := iptables.RemoveExistingChain(DockerChain, iptables.Filter); err != nil {
				logrus.Warnf("failed on removing iptables FILTER chain on cleanup: %v", err)
			}
		}
	}()

	isolationChain, err := iptables.NewChain(IsolationChain, iptables.Filter, false)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
	}

	if err := addReturnRule(IsolationChain); err != nil {
		return nil, nil, nil, err
	}

	return natChain, filterChain, isolationChain, nil
}

func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInterface) error {
	d := n.driver
	d.Lock()
	driverConfig := d.config
	d.Unlock()

	// Sanity check.
	if driverConfig.EnableIPTables == false {
		return fmt.Errorf("Cannot program chains, EnableIPTable is disabled")
	}

	// Pickup this configuraton option from driver
	hairpinMode := !driverConfig.EnableUserlandProxy

	addrv4, _, err := netutils.GetIfaceAddr(config.BridgeName)
	if err != nil {
		return fmt.Errorf("Failed to setup IP tables, cannot acquire Interface address: %s", err.Error())
	}
	ipnet := addrv4.(*net.IPNet)
	maskedAddrv4 := &net.IPNet{
		IP:   ipnet.IP.Mask(ipnet.Mask),
		Mask: ipnet.Mask,
	}
	if config.Internal {
		if err = setupInternalNetworkRules(config.BridgeName, maskedAddrv4, true); err != nil {
			return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
		}
		n.registerIptCleanFunc(func() error {
			return setupInternalNetworkRules(config.BridgeName, maskedAddrv4, false)
		})
	} else {
		if err = setupIPTablesInternal(config.BridgeName, maskedAddrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, true); err != nil {
			return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
		}
		n.registerIptCleanFunc(func() error {
			return setupIPTablesInternal(config.BridgeName, maskedAddrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, false)
		})
		natChain, filterChain, _, err := n.getDriverChains()
		if err != nil {
			return fmt.Errorf("Failed to setup IP tables, cannot acquire chain info %s", err.Error())
		}

		err = iptables.ProgramChain(natChain, config.BridgeName, hairpinMode, true)
		if err != nil {
			return fmt.Errorf("Failed to program NAT chain: %s", err.Error())
		}

		err = iptables.ProgramChain(filterChain, config.BridgeName, hairpinMode, true)
		if err != nil {
			return fmt.Errorf("Failed to program FILTER chain: %s", err.Error())
		}

		n.registerIptCleanFunc(func() error {
			return iptables.ProgramChain(filterChain, config.BridgeName, hairpinMode, false)
		})

		n.portMapper.SetIptablesChain(filterChain, n.getNetworkBridgeName())
	}

	if err := ensureJumpRule("FORWARD", IsolationChain); err != nil {
		return err
	}

	return nil
}

type iptRule struct {
	table   iptables.Table
	chain   string
	preArgs []string
	args    []string
}

func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairpin, enable bool) error {

	var (
		address   = addr.String()
		natRule   = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}}
		hpNatRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}}
		skipDNAT  = iptRule{table: iptables.Nat, chain: DockerChain, preArgs: []string{"-t", "nat"}, args: []string{"-i", bridgeIface, "-j", "RETURN"}}
		outRule   = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
		inRule    = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}
	)

	// Set NAT.
	if ipmasq {
		if err := programChainRule(natRule, "NAT", enable); err != nil {
			return err
		}
		if err := programChainRule(skipDNAT, "SKIP DNAT", enable); err != nil {
			return err
		}
	}

	// In hairpin mode, masquerade traffic from localhost
	if hairpin {
		if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable); err != nil {
			return err
		}
	}

	// Set Inter Container Communication.
	if err := setIcc(bridgeIface, icc, enable); err != nil {
		return err
	}

	// Set Accept on all non-intercontainer outgoing packets.
	if err := programChainRule(outRule, "ACCEPT NON_ICC OUTGOING", enable); err != nil {
		return err
	}

	// Set Accept on incoming packets for existing connections.
	if err := programChainRule(inRule, "ACCEPT INCOMING", enable); err != nil {
		return err
	}

	return nil
}

func programChainRule(rule iptRule, ruleDescr string, insert bool) error {
	var (
		prefix    []string
		operation string
		condition bool
		doesExist = iptables.Exists(rule.table, rule.chain, rule.args...)
	)

	if insert {
		condition = !doesExist
		prefix = []string{"-I", rule.chain}
		operation = "enable"
	} else {
		condition = doesExist
		prefix = []string{"-D", rule.chain}
		operation = "disable"
	}
	if rule.preArgs != nil {
		prefix = append(rule.preArgs, prefix...)
	}

	if condition {
		if err := iptables.RawCombinedOutput(append(prefix, rule.args...)...); err != nil {
			return fmt.Errorf("Unable to %s %s rule: %s", operation, ruleDescr, err.Error())
		}
	}

	return nil
}

func setIcc(bridgeIface string, iccEnable, insert bool) error {
	var (
		table      = iptables.Filter
		chain      = "FORWARD"
		args       = []string{"-i", bridgeIface, "-o", bridgeIface, "-j"}
		acceptArgs = append(args, "ACCEPT")
		dropArgs   = append(args, "DROP")
	)

	if insert {
		if !iccEnable {
			iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)

			if !iptables.Exists(table, chain, dropArgs...) {
				if err := iptables.RawCombinedOutput(append([]string{"-A", chain}, dropArgs...)...); err != nil {
					return fmt.Errorf("Unable to prevent intercontainer communication: %s", err.Error())
				}
			}
		} else {
			iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)

			if !iptables.Exists(table, chain, acceptArgs...) {
				if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, acceptArgs...)...); err != nil {
					return fmt.Errorf("Unable to allow intercontainer communication: %s", err.Error())
				}
			}
		}
	} else {
		// Remove any ICC rule.
		if !iccEnable {
			if iptables.Exists(table, chain, dropArgs...) {
				iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)
			}
		} else {
			if iptables.Exists(table, chain, acceptArgs...) {
				iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)
			}
		}
	}

	return nil
}

// Control Inter Network Communication. Install/remove only if it is not/is present.
func setINC(iface1, iface2 string, enable bool) error {
	var (
		table = iptables.Filter
		chain = IsolationChain
		args  = [2][]string{{"-i", iface1, "-o", iface2, "-j", "DROP"}, {"-i", iface2, "-o", iface1, "-j", "DROP"}}
	)

	if enable {
		for i := 0; i < 2; i++ {
			if iptables.Exists(table, chain, args[i]...) {
				continue
			}
			if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args[i]...)...); err != nil {
				return fmt.Errorf("unable to add inter-network communication rule: %v", err)
			}
		}
	} else {
		for i := 0; i < 2; i++ {
			if !iptables.Exists(table, chain, args[i]...) {
				continue
			}
			if err := iptables.RawCombinedOutput(append([]string{"-D", chain}, args[i]...)...); err != nil {
				return fmt.Errorf("unable to remove inter-network communication rule: %v", err)
			}
		}
	}

	return nil
}

func addReturnRule(chain string) error {
	var (
		table = iptables.Filter
		args  = []string{"-j", "RETURN"}
	)

	if iptables.Exists(table, chain, args...) {
		return nil
	}

	err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args...)...)
	if err != nil {
		return fmt.Errorf("unable to add return rule in %s chain: %s", chain, err.Error())
	}

	return nil
}

// Ensure the jump rule is on top
func ensureJumpRule(fromChain, toChain string) error {
	var (
		table = iptables.Filter
		args  = []string{"-j", toChain}
	)

	if iptables.Exists(table, fromChain, args...) {
		err := iptables.RawCombinedOutput(append([]string{"-D", fromChain}, args...)...)
		if err != nil {
			return fmt.Errorf("unable to remove jump to %s rule in %s chain: %s", toChain, fromChain, err.Error())
		}
	}

	err := iptables.RawCombinedOutput(append([]string{"-I", fromChain}, args...)...)
	if err != nil {
		return fmt.Errorf("unable to insert jump to %s rule in %s chain: %s", toChain, fromChain, err.Error())
	}

	return nil
}

func removeIPChains() {
	for _, chainInfo := range []iptables.ChainInfo{
		{Name: DockerChain, Table: iptables.Nat},
		{Name: DockerChain, Table: iptables.Filter},
		{Name: IsolationChain, Table: iptables.Filter},
	} {
		if err := chainInfo.Remove(); err != nil {
			logrus.Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err)
		}
	}
}

func setupInternalNetworkRules(bridgeIface string, addr net.Addr, insert bool) error {
	var (
		inDropRule  = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
		outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
	)
	if err := programChainRule(inDropRule, "DROP INCOMING", insert); err != nil {
		return err
	}
	if err := programChainRule(outDropRule, "DROP OUTGOING", insert); err != nil {
		return err
	}
	return nil
}
<commit_msg>Register natChain with portmapper<commit_after>package bridge

import (
	"fmt"
	"net"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/iptables"
	"github.com/docker/libnetwork/netutils"
)

// DockerChain: DOCKER iptable chain name
const (
	DockerChain    = "DOCKER"
	IsolationChain = "DOCKER-ISOLATION"
)

func setupIPChains(config *configuration) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
	// Sanity check.
	if config.EnableIPTables == false {
		return nil, nil, nil, fmt.Errorf("cannot create new chains, EnableIPTable is disabled")
	}

	hairpinMode := !config.EnableUserlandProxy

	natChain, err := iptables.NewChain(DockerChain, iptables.Nat, hairpinMode)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to create NAT chain: %v", err)
	}
	defer func() {
		if err != nil {
			if err := iptables.RemoveExistingChain(DockerChain, iptables.Nat); err != nil {
				logrus.Warnf("failed on removing iptables NAT chain on cleanup: %v", err)
			}
		}
	}()

	filterChain, err := iptables.NewChain(DockerChain, iptables.Filter, false)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to create FILTER chain: %v", err)
	}
	defer func() {
		if err != nil {
			if err := iptables.RemoveExistingChain(DockerChain, iptables.Filter); err != nil {
				logrus.Warnf("failed on removing iptables FILTER chain on cleanup: %v", err)
			}
		}
	}()

	isolationChain, err := iptables.NewChain(IsolationChain, iptables.Filter, false)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
	}

	if err := addReturnRule(IsolationChain); err != nil {
		return nil, nil, nil, err
	}

	return natChain, filterChain, isolationChain, nil
}

func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInterface) error {
	d := n.driver
	d.Lock()
	driverConfig := d.config
	d.Unlock()

	// Sanity check.
	if driverConfig.EnableIPTables == false {
		return fmt.Errorf("Cannot program chains, EnableIPTable is disabled")
	}

	// Pickup this configuraton option from driver
	hairpinMode := !driverConfig.EnableUserlandProxy

	addrv4, _, err := netutils.GetIfaceAddr(config.BridgeName)
	if err != nil {
		return fmt.Errorf("Failed to setup IP tables, cannot acquire Interface address: %s", err.Error())
	}
	ipnet := addrv4.(*net.IPNet)
	maskedAddrv4 := &net.IPNet{
		IP:   ipnet.IP.Mask(ipnet.Mask),
		Mask: ipnet.Mask,
	}
	if config.Internal {
		if err = setupInternalNetworkRules(config.BridgeName, maskedAddrv4, true); err != nil {
			return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
		}
		n.registerIptCleanFunc(func() error {
			return setupInternalNetworkRules(config.BridgeName, maskedAddrv4, false)
		})
	} else {
		if err = setupIPTablesInternal(config.BridgeName, maskedAddrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, true); err != nil {
			return fmt.Errorf("Failed to Setup IP tables: %s", err.Error())
		}
		n.registerIptCleanFunc(func() error {
			return setupIPTablesInternal(config.BridgeName, maskedAddrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, false)
		})
		natChain, filterChain, _, err := n.getDriverChains()
		if err != nil {
			return fmt.Errorf("Failed to setup IP tables, cannot acquire chain info %s", err.Error())
		}

		err = iptables.ProgramChain(natChain, config.BridgeName, hairpinMode, true)
		if err != nil {
			return fmt.Errorf("Failed to program NAT chain: %s", err.Error())
		}

		err = iptables.ProgramChain(filterChain, config.BridgeName, hairpinMode, true)
		if err != nil {
			return fmt.Errorf("Failed to program FILTER chain: %s", err.Error())
		}

		n.registerIptCleanFunc(func() error {
			return iptables.ProgramChain(filterChain, config.BridgeName, hairpinMode, false)
		})

		n.portMapper.SetIptablesChain(natChain, n.getNetworkBridgeName())
	}

	if err := ensureJumpRule("FORWARD", IsolationChain); err != nil {
		return err
	}

	return nil
}

type iptRule struct {
	table   iptables.Table
	chain   string
	preArgs []string
	args    []string
}

func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairpin, enable bool) error {

	var (
		address   = addr.String()
		natRule   = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}}
		hpNatRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}}
		skipDNAT  = iptRule{table: iptables.Nat, chain: DockerChain, preArgs: []string{"-t", "nat"}, args: []string{"-i", bridgeIface, "-j", "RETURN"}}
		outRule   = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
		inRule    = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}
	)

	// Set NAT.
	if ipmasq {
		if err := programChainRule(natRule, "NAT", enable); err != nil {
			return err
		}
	}

	if ipmasq && !hairpin {
		if err := programChainRule(skipDNAT, "SKIP DNAT", enable); err != nil {
			return err
		}
	}

	// In hairpin mode, masquerade traffic from localhost
	if hairpin {
		if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable); err != nil {
			return err
		}
	}

	// Set Inter Container Communication.
	if err := setIcc(bridgeIface, icc, enable); err != nil {
		return err
	}

	// Set Accept on all non-intercontainer outgoing packets.
	if err := programChainRule(outRule, "ACCEPT NON_ICC OUTGOING", enable); err != nil {
		return err
	}

	// Set Accept on incoming packets for existing connections.
	if err := programChainRule(inRule, "ACCEPT INCOMING", enable); err != nil {
		return err
	}

	return nil
}

func programChainRule(rule iptRule, ruleDescr string, insert bool) error {
	var (
		prefix    []string
		operation string
		condition bool
		doesExist = iptables.Exists(rule.table, rule.chain, rule.args...)
	)

	if insert {
		condition = !doesExist
		prefix = []string{"-I", rule.chain}
		operation = "enable"
	} else {
		condition = doesExist
		prefix = []string{"-D", rule.chain}
		operation = "disable"
	}
	if rule.preArgs != nil {
		prefix = append(rule.preArgs, prefix...)
	}

	if condition {
		if err := iptables.RawCombinedOutput(append(prefix, rule.args...)...); err != nil {
			return fmt.Errorf("Unable to %s %s rule: %s", operation, ruleDescr, err.Error())
		}
	}

	return nil
}

func setIcc(bridgeIface string, iccEnable, insert bool) error {
	var (
		table      = iptables.Filter
		chain      = "FORWARD"
		args       = []string{"-i", bridgeIface, "-o", bridgeIface, "-j"}
		acceptArgs = append(args, "ACCEPT")
		dropArgs   = append(args, "DROP")
	)

	if insert {
		if !iccEnable {
			iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)

			if !iptables.Exists(table, chain, dropArgs...) {
				if err := iptables.RawCombinedOutput(append([]string{"-A", chain}, dropArgs...)...); err != nil {
					return fmt.Errorf("Unable to prevent intercontainer communication: %s", err.Error())
				}
			}
		} else {
			iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)

			if !iptables.Exists(table, chain, acceptArgs...) {
				if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, acceptArgs...)...); err != nil {
					return fmt.Errorf("Unable to allow intercontainer communication: %s", err.Error())
				}
			}
		}
	} else {
		// Remove any ICC rule.
		if !iccEnable {
			if iptables.Exists(table, chain, dropArgs...) {
				iptables.Raw(append([]string{"-D", chain}, dropArgs...)...)
			}
		} else {
			if iptables.Exists(table, chain, acceptArgs...) {
				iptables.Raw(append([]string{"-D", chain}, acceptArgs...)...)
			}
		}
	}

	return nil
}

// Control Inter Network Communication. Install/remove only if it is not/is present.
func setINC(iface1, iface2 string, enable bool) error {
	var (
		table = iptables.Filter
		chain = IsolationChain
		args  = [2][]string{{"-i", iface1, "-o", iface2, "-j", "DROP"}, {"-i", iface2, "-o", iface1, "-j", "DROP"}}
	)

	if enable {
		for i := 0; i < 2; i++ {
			if iptables.Exists(table, chain, args[i]...) {
				continue
			}
			if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args[i]...)...); err != nil {
				return fmt.Errorf("unable to add inter-network communication rule: %v", err)
			}
		}
	} else {
		for i := 0; i < 2; i++ {
			if !iptables.Exists(table, chain, args[i]...) {
				continue
			}
			if err := iptables.RawCombinedOutput(append([]string{"-D", chain}, args[i]...)...); err != nil {
				return fmt.Errorf("unable to remove inter-network communication rule: %v", err)
			}
		}
	}

	return nil
}

func addReturnRule(chain string) error {
	var (
		table = iptables.Filter
		args  = []string{"-j", "RETURN"}
	)

	if iptables.Exists(table, chain, args...) {
		return nil
	}

	err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args...)...)
	if err != nil {
		return fmt.Errorf("unable to add return rule in %s chain: %s", chain, err.Error())
	}

	return nil
}

// Ensure the jump rule is on top
func ensureJumpRule(fromChain, toChain string) error {
	var (
		table = iptables.Filter
		args  = []string{"-j", toChain}
	)

	if iptables.Exists(table, fromChain, args...) {
		err := iptables.RawCombinedOutput(append([]string{"-D", fromChain}, args...)...)
		if err != nil {
			return fmt.Errorf("unable to remove jump to %s rule in %s chain: %s", toChain, fromChain, err.Error())
		}
	}

	err := iptables.RawCombinedOutput(append([]string{"-I", fromChain}, args...)...)
	if err != nil {
		return fmt.Errorf("unable to insert jump to %s rule in %s chain: %s", toChain, fromChain, err.Error())
	}

	return nil
}

func removeIPChains() {
	for _, chainInfo := range []iptables.ChainInfo{
		{Name: DockerChain, Table: iptables.Nat},
		{Name: DockerChain, Table: iptables.Filter},
		{Name: IsolationChain, Table: iptables.Filter},
	} {
		if err := chainInfo.Remove(); err != nil {
			logrus.Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err)
		}
	}
}

func setupInternalNetworkRules(bridgeIface string, addr net.Addr, insert bool) error {
	var (
		inDropRule  = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
		outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
	)
	if err := programChainRule(inDropRule, "DROP INCOMING", insert); err != nil {
		return err
	}
	if err := programChainRule(outDropRule, "DROP OUTGOING", insert); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package game

import (
	"github.com/ghthor/engine/rpg2d/coord"
	"github.com/ghthor/engine/rpg2d/quad"
	"github.com/ghthor/engine/sim/stime"

	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
)

func DescribeCollision(c gospec.Context) {
	cell := func(x, y int) coord.Cell { return coord.Cell{x, y} }
	pa := func(start, speed int64, origin, dest coord.Cell) *coord.PathAction {
		return &coord.PathAction{
			Span: stime.NewSpan(stime.Time(start), stime.Time(start+speed)),
			Orig: origin,
			Dest: dest,
		}
	}

	c.Specify("a collision between", func() {
		c.Specify("2 actors", func() {
			c.Specify("that are both moving", func() {
				c.Specify("in the same direction", func() {
					pa0 := pa(0, 5, cell(0, 0), cell(0, 1))
					pa1 := pa(0, 10, cell(0, 1), cell(0, 2))

					index := actorIndex{
						0: &actor{
							actorEntity: actorEntity{
								id: 0,

								cell:   pa0.Orig,
								facing: pa0.Direction(),
							},
						},

						1: &actor{
							actorEntity: actorEntity{
								id: 1,

								cell:   pa1.Orig,
								facing: pa1.Direction(),
							},
						},
					}

					index[0].applyPathAction(pa0)
					index[1].applyPathAction(pa1)

					phase := narrowPhase{index}

					testCases := []struct {
						spec string
						cgrp quad.CollisionGroup
					}{{
						"AB", quad.CollisionGroup{}.AddCollision(quad.Collision{
							index[0].Entity(),
							index[1].Entity(),
						}),
					}, {
						"BA", quad.CollisionGroup{}.AddCollision(quad.Collision{
							index[1].Entity(),
							index[0].Entity(),
						}),
					}}

					for _, testCase := range testCases {
						c.Specify(testCase.spec, func() {
							stillExisting, removed := phase.ResolveCollisions(&testCase.cgrp, 0)
							c.Assume(len(stillExisting), Equals, 2)
							c.Assume(len(removed), Equals, 0)

							c.Expect(index[0].pathAction, IsNil)
							c.Expect(*index[1].pathAction, Equals, *pa1)
						})
					}
				})
			})
		})
	})
}
<commit_msg>[game] refactor collision specifications<commit_after>package game

import (
	"github.com/ghthor/engine/rpg2d/coord"
	"github.com/ghthor/engine/rpg2d/quad"
	"github.com/ghthor/engine/sim/stime"

	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
)

type spec_2moving struct {
	spec         string
	paths        []coord.PathAction
	expectations func(spec_2moving, actorIndex, gospec.Context)
}

func (t spec_2moving) runSpec(c gospec.Context) {
	pa0 := t.paths[0]
	pa1 := t.paths[1]

	index := actorIndex{
		0: &actor{
			actorEntity: actorEntity{
				id: 0,

				cell:   pa0.Orig,
				facing: pa0.Direction(),
			},
		},

		1: &actor{
			actorEntity: actorEntity{
				id: 1,

				cell:   pa1.Orig,
				facing: pa1.Direction(),
			},
		},
	}

	index[0].applyPathAction(&pa0)
	index[1].applyPathAction(&pa1)

	phase := narrowPhase{index}
	testCases := []struct {
		spec string
		cgrp quad.CollisionGroup
	}{{
		"AB", quad.CollisionGroup{}.AddCollision(quad.Collision{
			index[0].Entity(),
			index[1].Entity(),
		}),
	}, {
		"BA", quad.CollisionGroup{}.AddCollision(quad.Collision{
			index[1].Entity(),
			index[0].Entity(),
		}),
	}}

	c.Specify(t.spec, func() {
		for _, testCase := range testCases {
			c.Specify(testCase.spec, func() {
				stillExisting, removed := phase.ResolveCollisions(&testCase.cgrp, 0)
				c.Assume(len(stillExisting), Equals, 2)
				c.Assume(len(removed), Equals, 0)

				t.expectations(t, index, c)
			})
		}
	})
}

func DescribeCollision(c gospec.Context) {
	cell := func(x, y int) coord.Cell { return coord.Cell{x, y} }
	pa := func(start, speed int64, origin, dest coord.Cell) coord.PathAction {
		return coord.PathAction{
			Span: stime.NewSpan(stime.Time(start), stime.Time(start+speed)),
			Orig: origin,
			Dest: dest,
		}
	}

	c.Specify("a collision between", func() {
		c.Specify("2 actors", func() {

			c.Specify("that are both moving", func() {
				testCases := []spec_2moving{{
					spec: "in the same direction",
					paths: []coord.PathAction{
						pa(0, 5, cell(0, 0), cell(0, 1)),
						pa(0, 10, cell(0, 1), cell(0, 2)),
					},
					expectations: func(testCase spec_2moving, index actorIndex, c gospec.Context) {
						c.Expect(index[0].pathAction, IsNil)
						c.Expect(*index[1].pathAction, Equals, testCase.paths[1])
					}},
				}

				for _, testCase := range testCases {
					testCase.runSpec(c)
				}
			})
		})
	})
}
<|endoftext|>


<commit_before>package extensions

import (
	"encoding/xml"
	"fmt"
	"github.com/pawelszydlo/papa-bot"
	"net/url"
	"strings"
	"github.com/pawelszydlo/papa-bot/events"
)

/*
ExtensionWolfram - query Wolfram Alpha.

Used custom variables:
- WolframKey - Your Wolfram Alpha AppID key.
*/
type ExtensionWolfram struct {
	announced map[string]bool
	Texts     *extensionWolframTexts
	bot       *papaBot.Bot
}

type extensionWolframTexts struct {
	NoResult string
	Or       string
}

// Structs for Wolfram responses.
type queryResult struct {
	Success bool  `xml:"success,attr"`
	Error   bool  `xml:"error,attr"`
	Pods    []pod `xml:"pod"`
}

type pod struct {
	Title   string `xml:"title,attr"`
	Scanner string `xml:"scanner,attr"`
	Id      string `xml:"id,attr"`

	Subpods []subPod `xml:"subpod"`
}

type subPod struct {
	Title     string `xml:"title,attr"`
	PlainText string `xml:"plaintext"`
}

// Init inits the extension.
func (ext *ExtensionWolfram) Init(bot *papaBot.Bot) error {
	// Init variables.
	ext.announced = map[string]bool{}
	// Load texts.
	texts := &extensionWolframTexts{}
	if err := bot.LoadTexts("wolfram", texts); err != nil {
		return err
	}
	ext.Texts = texts
	// Register new command.
	bot.RegisterCommand(&papaBot.BotCommand{
		[]string{"wa", "wolfram"},
		false, false, false,
		"<query>", "Search Wolfram Alpha for <query>.",
		ext.commandWolfram})
	ext.bot = bot
	return nil
}

func (ext *ExtensionWolfram) queryWolfram(query string, format events.Formatting) string {
	appId := ext.bot.GetVar("WolframKey")
	if appId == "" {
		ext.bot.Log.Error("Wolfram Alpha AppID key not set! Set the 'WolframKey' variable in the bot.")
		return ""
	}

	err, _, body := ext.bot.GetPageBody(
		fmt.Sprintf(
			"http://api.wolframalpha.com/v2/query?format=plaintext&appid=%s&podindex=1,2&input=%s",
			appId, strings.Replace(url.QueryEscape(query), "+", "%20", -1),
		), nil)
	if err != nil {
		ext.bot.Log.Warningf("Error getting Wolfram data: %s", err)
		return ""
	}

	// Parse XML.
	data := new(queryResult)
	if err := xml.Unmarshal(body, &data); err != nil {
		ext.bot.Log.Errorf("Error parsing Wolfram data: %s", err)
		return ""
	}

	// Did the query succeed?
	if data.Error || !data.Success {
		return ext.Texts.NoResult
	}

	input := ""
	result := []string{} // Result can have multiple forms.

	for _, pod := range data.Pods {
		// Get the input interpretation.
		if pod.Id == "Input" {
			for _, subpod := range pod.Subpods {
				input = subpod.PlainText
			}
		}
		// Get the result.
		if pod.Id == "Result" || pod.Id == "Value" {
			for _, subpod := range pod.Subpods {
				if format == events.FormatMarkdown {
					result = append(result, "**"+subpod.PlainText+"**")
				} else if format == events.FormatIRC {
					result = append(result, "\x0308"+subpod.PlainText+"\x03")
				} else {
					result = append(result, subpod.PlainText)
				}
			}
		}
	}

	return fmt.Sprintf("%s = %s", input, strings.Join(result, ext.Texts.Or))
}

// commandMovie is a command for manually searching for movies.
func (ext *ExtensionWolfram) commandWolfram(bot *papaBot.Bot, sourceEvent *events.EventMessage, params []string) {
	if len(params) < 1 {
		return
	}
	search := strings.Join(params, " ")

	// Announce each article only once.
	if ext.announced[sourceEvent.Channel+search] {
		return
	}

	content := ext.queryWolfram(search, sourceEvent.TransportFormatting)

	// Error occured.
	if content == "" {
		return
	}

	maxLen := 300
	if sourceEvent.TransportFormatting == events.FormatMarkdown {
		maxLen = 3000
	}

	// Announce.
	contentPreview := content
	contentFull := ""
	if len(content) > maxLen {
		contentPreview = content[:maxLen] + "…"
		contentFull = content
	}

	notice := fmt.Sprintf("%s, %s", sourceEvent.Nick, contentPreview)
	bot.SendMessage(sourceEvent, notice)
	ext.announced[sourceEvent.Channel+search] = true

	if contentFull != "" {
		bot.AddMoreInfo(sourceEvent.TransportName, sourceEvent.Channel, contentFull)
	}
}
<commit_msg>WolframAlpha result caching.<commit_after>package extensions

import (
	"encoding/xml"
	"fmt"
	"github.com/pawelszydlo/papa-bot"
	"net/url"
	"strings"
	"github.com/pawelszydlo/papa-bot/events"
)

/*
ExtensionWolfram - query Wolfram Alpha.

Used custom variables:
- WolframKey - Your Wolfram Alpha AppID key.
*/
type ExtensionWolfram struct {
	announced map[string]string
	Texts     *extensionWolframTexts
	bot       *papaBot.Bot
}

type extensionWolframTexts struct {
	NoResult string
	Or       string
}

// Structs for Wolfram responses.
type queryResult struct {
	Success bool  `xml:"success,attr"`
	Error   bool  `xml:"error,attr"`
	Pods    []pod `xml:"pod"`
}

type pod struct {
	Title   string `xml:"title,attr"`
	Scanner string `xml:"scanner,attr"`
	Id      string `xml:"id,attr"`

	Subpods []subPod `xml:"subpod"`
}

type subPod struct {
	Title     string `xml:"title,attr"`
	PlainText string `xml:"plaintext"`
}

// Init inits the extension.
func (ext *ExtensionWolfram) Init(bot *papaBot.Bot) error {
	// Init variables.
	ext.announced = map[string]string{}
	// Load texts.
	texts := &extensionWolframTexts{}
	if err := bot.LoadTexts("wolfram", texts); err != nil {
		return err
	}
	ext.Texts = texts
	// Register new command.
	bot.RegisterCommand(&papaBot.BotCommand{
		[]string{"wa", "wolfram"},
		false, false, false,
		"<query>", "Search Wolfram Alpha for <query>.",
		ext.commandWolfram})
	ext.bot = bot
	return nil
}

func (ext *ExtensionWolfram) queryWolfram(query string, format events.Formatting) string {
	appId := ext.bot.GetVar("WolframKey")
	if appId == "" {
		ext.bot.Log.Error("Wolfram Alpha AppID key not set! Set the 'WolframKey' variable in the bot.")
		return ""
	}

	ext.bot.Log.Debugf("Querying WolframAlpha for %s...", query)
	err, _, body := ext.bot.GetPageBody(
		fmt.Sprintf(
			"http://api.wolframalpha.com/v2/query?format=plaintext&appid=%s&podindex=1,2&input=%s",
			appId, strings.Replace(url.QueryEscape(query), "+", "%20", -1),
		), nil)
	if err != nil {
		ext.bot.Log.Warningf("Error getting Wolfram data: %s", err)
		return ""
	}

	// Parse XML.
	data := new(queryResult)
	if err := xml.Unmarshal(body, &data); err != nil {
		ext.bot.Log.Errorf("Error parsing Wolfram data: %s", err)
		return ""
	}

	// Did the query succeed?
	if data.Error || !data.Success {
		return ext.Texts.NoResult
	}

	input := ""
	result := []string{} // Result can have multiple forms.

	for _, pod := range data.Pods {
		// Get the input interpretation.
		if pod.Id == "Input" {
			for _, subpod := range pod.Subpods {
				input = subpod.PlainText
			}
		}
		// Get the result.
		if pod.Id == "Result" || pod.Id == "Value" {
			for _, subpod := range pod.Subpods {
				if format == events.FormatMarkdown {
					result = append(result, "**"+subpod.PlainText+"**")
				} else if format == events.FormatIRC {
					result = append(result, "\x0308"+subpod.PlainText+"\x03")
				} else {
					result = append(result, subpod.PlainText)
				}
			}
		}
	}

	return fmt.Sprintf("%s = %s", input, strings.Join(result, ext.Texts.Or))
}

// commandMovie is a command for manually searching for movies.
func (ext *ExtensionWolfram) commandWolfram(bot *papaBot.Bot, sourceEvent *events.EventMessage, params []string) {
	if len(params) < 1 {
		return
	}
	search := strings.Join(params, " ")

	// Check if we have the result cached.
	if val, exists := ext.announced[sourceEvent.Channel+search]; exists {
		bot.SendMessage(sourceEvent, fmt.Sprintf("%s, %s", sourceEvent.Nick, val))
		return
	}

	content := ext.queryWolfram(search, sourceEvent.TransportFormatting)

	// Error occured.
	if content == "" {
		return
	}

	maxLen := 300
	if sourceEvent.TransportFormatting == events.FormatMarkdown {
		maxLen = 3000
	}

	// Announce.
	contentPreview := content
	contentFull := ""
	if len(content) > maxLen {
		contentPreview = content[:maxLen] + "…"
		contentFull = content
	}

	bot.SendMessage(sourceEvent, fmt.Sprintf("%s, %s", sourceEvent.Nick, contentPreview))
	ext.announced[sourceEvent.Channel+search] = contentPreview

	if contentFull != "" {
		bot.AddMoreInfo(sourceEvent.TransportName, sourceEvent.Channel, contentFull)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	cq_config "go.chromium.org/luci/cq/api/config/v2"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/cq"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/supported_branches"
	"go.skia.org/infra/go/util"
)

var (
	// Flags.
	branch         = flag.String("branch", "", "Name of the new branch, without refs/heads prefix.")
	deleteBranch   = flag.String("delete", "", "Name of an existing branch to delete, without refs/heads prefix.")
	excludeTrybots = common.NewMultiStringFlag("exclude-trybots", nil, "Regular expressions for trybot names to exclude.")
	owner          = flag.String("owner", "", "Owner of the new branch.")
	repoUrl        = flag.String("repo", common.REPO_SKIA, "URL of the git repository.")
	submit         = flag.Bool("submit", false, "If set, automatically submit the CL to update the CQ and supported branches.")
)

func main() {
	common.Init()

	if *branch == "" {
		sklog.Fatal("--branch is required.")
	}
	newRef := fmt.Sprintf("refs/heads/%s", *branch)
	if *owner == "" {
		sklog.Fatal("--owner is required.")
	}
	excludeTrybotRegexp := make([]*regexp.Regexp, 0, len(*excludeTrybots))
	for _, excludeTrybot := range *excludeTrybots {
		re, err := regexp.Compile(excludeTrybot)
		if err != nil {
			sklog.Fatalf("Failed to compile regular expression from %q; %s", excludeTrybot, err)
		}
		excludeTrybotRegexp = append(excludeTrybotRegexp, re)
	}

	// Setup.
	wd, err := ioutil.TempDir("", "new-branch")
	if err != nil {
		sklog.Fatal(err)
	}
	defer util.RemoveAll(wd)

	ts, err := auth.NewDefaultTokenSource(true, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatal(err)
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
	gUrl := strings.Split(*repoUrl, ".googlesource.com")[0] + "-review.googlesource.com"
	g, err := gerrit.NewGerrit(gUrl, client)
	if err != nil {
		sklog.Fatal(err)
	}
	repo := gitiles.NewRepo(*repoUrl, client)
	ctx := context.Background()
	baseCommitInfo, err := repo.Details(ctx, cq.CQ_CFG_REF)
	if err != nil {
		sklog.Fatal(err)
	}
	baseCommit := baseCommitInfo.Hash

	// Download the CQ config file and modify it.
	var buf bytes.Buffer
	if err := repo.ReadFileAtRef(ctx, cq.CQ_CFG_FILE, baseCommit, &buf); err != nil {
		sklog.Fatal(err)
	}
	newCfgBytes, err := cq.WithUpdateCQConfig(buf.Bytes(), func(cfg *cq_config.Config) error {
		cg, _, _, err := cq.MatchConfigGroup(cfg, newRef)
		if err != nil {
			return err
		}
		if cg != nil {
			_, _ = fmt.Fprintf(os.Stderr, "Already have %s in %s; not adding a duplicate.\n", newRef, cq.CQ_CFG_FILE)
		} else {
			if err := cq.CloneBranch(cfg, "master", *branch, false, false, excludeTrybotRegexp); err != nil {
				return err
			}
		}
		if *deleteBranch != "" {
			if err := cq.DeleteBranch(cfg, *deleteBranch); err != nil {
				return err
			}
		}
		return nil
	})

	// Download and modify the supported-branches.json file.
	buf = bytes.Buffer{}
	if err := repo.ReadFileAtRef(ctx, supported_branches.SUPPORTED_BRANCHES_FILE, baseCommit, &buf); err != nil {
		sklog.Fatal(err)
	}
	sbc, err := supported_branches.DecodeConfig(&buf)
	if err != nil {
		sklog.Fatal(err)
	}
	deleteRef := ""
	if *deleteBranch != "" {
		deleteRef = fmt.Sprintf("refs/heads/%s", *deleteBranch)
	}
	foundNewRef := false
	newBranches := make([]*supported_branches.SupportedBranch, 0, len(sbc.Branches)+1)
	for _, sb := range sbc.Branches {
		if deleteRef == "" || deleteRef != sb.Ref {
			newBranches = append(newBranches, sb)
		}
		if sb.Ref == newRef {
			foundNewRef = true
		}
	}
	if foundNewRef {
		_, _ = fmt.Fprintf(os.Stderr, "Already have %s in %s; not adding a duplicate.\n", newRef, supported_branches.SUPPORTED_BRANCHES_FILE)
	} else {
		newBranches = append(newBranches, &supported_branches.SupportedBranch{
			Ref:   newRef,
			Owner: *owner,
		})
	}
	sbc.Branches = newBranches
	buf = bytes.Buffer{}
	if err := supported_branches.EncodeConfig(&buf, sbc); err != nil {
		sklog.Fatal(err)
	}

	// Create the Gerrit CL.
	commitMsg := fmt.Sprintf("Add supported branch %s", *branch)
	if *deleteBranch != "" {
		commitMsg += fmt.Sprintf(", remove %s", *deleteBranch)
	}
	repoSplit := strings.Split(*repoUrl, "/")
	project := strings.TrimSuffix(repoSplit[len(repoSplit)-1], ".git")
	ci, err := gerrit.CreateAndEditChange(ctx, g, project, cq.CQ_CFG_REF, commitMsg, baseCommit, func(ctx context.Context, g gerrit.GerritInterface, ci *gerrit.ChangeInfo) error {
		if err := g.EditFile(ctx, ci, cq.CQ_CFG_FILE, string(newCfgBytes)); err != nil {
			return err
		}
		if err := g.EditFile(ctx, ci, supported_branches.SUPPORTED_BRANCHES_FILE, string(buf.Bytes())); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		sklog.Fatal(err)
	}
	fmt.Println(fmt.Sprintf("Uploaded change https://skia-review.googlesource.com/%d", ci.Issue))
	if *submit {
		if err := g.SetReadyForReview(ctx, ci); err != nil {
			sklog.Fatalf("Failed to set ready for review: %s", err)
		}
		if err := g.SetReview(ctx, ci, "", gerrit.CONFIG_CHROMIUM.SelfApproveLabels, nil); err != nil {
			sklog.Fatalf("Failed to set Code-Review+1: %s", err)
		}
		if err := g.Submit(ctx, ci); err != nil {
			sklog.Fatalf("Failed to submit CL: %s", err)
		}
	}
}
<commit_msg>[chrome branch] Only set ready-for-review if the CL is WIP<commit_after>package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	cq_config "go.chromium.org/luci/cq/api/config/v2"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/cq"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/supported_branches"
	"go.skia.org/infra/go/util"
)

var (
	// Flags.
	branch         = flag.String("branch", "", "Name of the new branch, without refs/heads prefix.")
	deleteBranch   = flag.String("delete", "", "Name of an existing branch to delete, without refs/heads prefix.")
	excludeTrybots = common.NewMultiStringFlag("exclude-trybots", nil, "Regular expressions for trybot names to exclude.")
	owner          = flag.String("owner", "", "Owner of the new branch.")
	repoUrl        = flag.String("repo", common.REPO_SKIA, "URL of the git repository.")
	submit         = flag.Bool("submit", false, "If set, automatically submit the CL to update the CQ and supported branches.")
)

func main() {
	common.Init()

	if *branch == "" {
		sklog.Fatal("--branch is required.")
	}
	newRef := fmt.Sprintf("refs/heads/%s", *branch)
	if *owner == "" {
		sklog.Fatal("--owner is required.")
	}
	excludeTrybotRegexp := make([]*regexp.Regexp, 0, len(*excludeTrybots))
	for _, excludeTrybot := range *excludeTrybots {
		re, err := regexp.Compile(excludeTrybot)
		if err != nil {
			sklog.Fatalf("Failed to compile regular expression from %q; %s", excludeTrybot, err)
		}
		excludeTrybotRegexp = append(excludeTrybotRegexp, re)
	}

	// Setup.
	wd, err := ioutil.TempDir("", "new-branch")
	if err != nil {
		sklog.Fatal(err)
	}
	defer util.RemoveAll(wd)

	ts, err := auth.NewDefaultTokenSource(true, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatal(err)
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
	gUrl := strings.Split(*repoUrl, ".googlesource.com")[0] + "-review.googlesource.com"
	g, err := gerrit.NewGerrit(gUrl, client)
	if err != nil {
		sklog.Fatal(err)
	}
	repo := gitiles.NewRepo(*repoUrl, client)
	ctx := context.Background()
	baseCommitInfo, err := repo.Details(ctx, cq.CQ_CFG_REF)
	if err != nil {
		sklog.Fatal(err)
	}
	baseCommit := baseCommitInfo.Hash

	// Download the CQ config file and modify it.
	var buf bytes.Buffer
	if err := repo.ReadFileAtRef(ctx, cq.CQ_CFG_FILE, baseCommit, &buf); err != nil {
		sklog.Fatal(err)
	}
	newCfgBytes, err := cq.WithUpdateCQConfig(buf.Bytes(), func(cfg *cq_config.Config) error {
		cg, _, _, err := cq.MatchConfigGroup(cfg, newRef)
		if err != nil {
			return err
		}
		if cg != nil {
			_, _ = fmt.Fprintf(os.Stderr, "Already have %s in %s; not adding a duplicate.\n", newRef, cq.CQ_CFG_FILE)
		} else {
			if err := cq.CloneBranch(cfg, "master", *branch, false, false, excludeTrybotRegexp); err != nil {
				return err
			}
		}
		if *deleteBranch != "" {
			if err := cq.DeleteBranch(cfg, *deleteBranch); err != nil {
				return err
			}
		}
		return nil
	})

	// Download and modify the supported-branches.json file.
	buf = bytes.Buffer{}
	if err := repo.ReadFileAtRef(ctx, supported_branches.SUPPORTED_BRANCHES_FILE, baseCommit, &buf); err != nil {
		sklog.Fatal(err)
	}
	sbc, err := supported_branches.DecodeConfig(&buf)
	if err != nil {
		sklog.Fatal(err)
	}
	deleteRef := ""
	if *deleteBranch != "" {
		deleteRef = fmt.Sprintf("refs/heads/%s", *deleteBranch)
	}
	foundNewRef := false
	newBranches := make([]*supported_branches.SupportedBranch, 0, len(sbc.Branches)+1)
	for _, sb := range sbc.Branches {
		if deleteRef == "" || deleteRef != sb.Ref {
			newBranches = append(newBranches, sb)
		}
		if sb.Ref == newRef {
			foundNewRef = true
		}
	}
	if foundNewRef {
		_, _ = fmt.Fprintf(os.Stderr, "Already have %s in %s; not adding a duplicate.\n", newRef, supported_branches.SUPPORTED_BRANCHES_FILE)
	} else {
		newBranches = append(newBranches, &supported_branches.SupportedBranch{
			Ref:   newRef,
			Owner: *owner,
		})
	}
	sbc.Branches = newBranches
	buf = bytes.Buffer{}
	if err := supported_branches.EncodeConfig(&buf, sbc); err != nil {
		sklog.Fatal(err)
	}

	// Create the Gerrit CL.
	commitMsg := fmt.Sprintf("Add supported branch %s", *branch)
	if *deleteBranch != "" {
		commitMsg += fmt.Sprintf(", remove %s", *deleteBranch)
	}
	repoSplit := strings.Split(*repoUrl, "/")
	project := strings.TrimSuffix(repoSplit[len(repoSplit)-1], ".git")
	ci, err := gerrit.CreateAndEditChange(ctx, g, project, cq.CQ_CFG_REF, commitMsg, baseCommit, func(ctx context.Context, g gerrit.GerritInterface, ci *gerrit.ChangeInfo) error {
		if err := g.EditFile(ctx, ci, cq.CQ_CFG_FILE, string(newCfgBytes)); err != nil {
			return err
		}
		if err := g.EditFile(ctx, ci, supported_branches.SUPPORTED_BRANCHES_FILE, string(buf.Bytes())); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		sklog.Fatal(err)
	}
	fmt.Println(fmt.Sprintf("Uploaded change https://skia-review.googlesource.com/%d", ci.Issue))
	if *submit {
		if ci.WorkInProgress {
			if err := g.SetReadyForReview(ctx, ci); err != nil {
				sklog.Fatalf("Failed to set ready for review: %s", err)
			}
		}
		if err := g.SetReview(ctx, ci, "", gerrit.CONFIG_CHROMIUM.SelfApproveLabels, nil); err != nil {
			sklog.Fatalf("Failed to set Code-Review+1: %s", err)
		}
		if err := g.Submit(ctx, ci); err != nil {
			sklog.Fatalf("Failed to submit CL: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>// geomi, 거미, is a basic spider for crawling specific webs. It was designed to crawl
// either a site, or a subset of a site, indexing the pages and where they link to.
// Any links that go outside of the provided base URL are noted but not followed.
//
//
// Even though geomi was designed with the intended use being crawling ones own site,
// or a client's site, it does come with some basic behavior configuration to make it
// a friendly bot:
//   * respects ROBOTS.txt TODO
//   * configurable concurrent walkers TODO
//   * configurable wait interval range TODO
//   * configurable max requests per: TODO
//     * min
//     * hour
//     * day
//     * month
//
// To start, go get the geomi package:
//    go get github.com/mohae/geomi
//
// Import it into your code:
//    import github.com/mohae/geomi
//
// Set up the site for the spider:
//   s := geomi.NewSite("http://example.com")
//
// The url that is passed to the NewSite() function is the base case, after which
// a BFS traversal is done.
//
package geomi

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/mohae/utilitybelt/queue"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// Fetcher is an interface that makes it easier to test. In the future, it may be
// useful for other purposes, but that would require exporting the method that uses
// it first.
type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page
	Fetch(url string) (body string, urls []string, err error)
}

//}
// fetched tracks URLs that have been (or are being) fetched.
// The lock must be held while reading from or writing to the map.
var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// a page is a url. This is usually some content wiht a number of elements, but it
// could be something as simple as the url for cat.jpg on your site and represent that
// image.
type Page struct {
	*url.URL
	distance int
	body     string
	links    []string // immediate children
}

// responseInfo contains the status and error information from a get
type responseInfo struct {
	Status     string
	StatusCode int
	Err        error
}

// Site is a type that implements fetcher
type Site struct {
	*url.URL
}

// Implements fetcher.
// TODO: make the design cleaner
func (s Site) Fetch(url string) (body string, urls []string, err error) {
	// see if the passed url is outside of the baseURL
	resp, err := http.Get(url)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()
	buff := &bytes.Buffer{}
	tee := io.TeeReader(resp.Body, buff)
	tokens := getTokens(tee)
	if len(tokens) == 0 {
		return "", nil, fmt.Errorf("%s: nothing in body", url)
	}
	urls, err = s.linksFromTokens(tokens)
	if err != nil {
		return "", nil, err
	}
	return buff.String(), urls, nil
}

// linksFromTokens returns a list of links (href a) found in the token slice
// TODO should internal links be tracked separatly? i.e. record them in a
// separate var (so they don't get fetched)
func (s *Site) linksFromTokens(tokens []html.Token) ([]string, error) {
	var links []string
	for _, token := range tokens {
		if token.Type == html.StartTagToken && token.DataAtom.String() == "a" {
			for _, attr := range token.Attr {
				// We only care about links that aren't #
				if attr.Key == "href" && !strings.HasPrefix(attr.Val, "#") {
					// parse the url
					url, err := url.Parse(attr.Val)
					if err != nil {
						return nil, err
					}
					link := s.URL.ResolveReference(url)
					links = append(links, link.String())
				}
			}
		}
	}
	return links, nil
}

// Spider crawls the target. It contains all information needed to manage the crawl
// including keeping track of work to do, work done, and the results.
type Spider struct {
	*queue.Queue
	sync.Mutex
	wg                 sync.WaitGroup
	*url.URL                  // the start url
	bot                string // the name of the bot for robots
	concurrency        int    // concurrency level of crawling
	fetchInterval      int64  // if > 0, interval, in milliseconds to wait between gets
	intervalJitter     int64  // the max jitter to be added, per fetch. Jitter is a rand with this as max.
	RestrictToScheme   bool   // if true, only crawls urls with the same scheme as baseURL
	RespectRobots      bool   // whether or not to respect the robots.txt
	CheckExternalLinks bool   // Check the status of any links that go to other domains
	robots             *robotstxt.Group
	maxDepth           int
	Pages              map[string]Page
	foundURLs          map[string]struct{}     // keeps track of urls found to prevent recrawling
	fetchedURLs        map[string]error        // urls that have been fetched with their status
	skippedURLs        map[string]struct{}     // urls within the same domain that are not retrieved
	externalHosts      map[string]struct{}     // list of external hosts TODO: elide?
	externalLinks      map[string]responseInfo // list of external links; if fetched,
}

// returns a Spider with the its site's baseUrl set. The baseUrl is the start point for
// the crawl. It is also the restriction on the crawl:
func NewSpider(start string) (*Spider, error) {
	if start == "" {
		return nil, errors.New("newSpider: the start url cannot be empty")
	}
	var err error
	spider := &Spider{Queue: queue.New(128, 0),
		bot:           "geomi",
		RespectRobots: true,
		Pages:         make(map[string]Page),
		foundURLs:     make(map[string]struct{}),
		fetchedURLs:   make(map[string]error),
		skippedURLs:   make(map[string]struct{}),
		externalHosts: make(map[string]struct{}),
		externalLinks: make(map[string]responseInfo),
	}
	spider.URL, err = url.Parse(start)
	if err != nil {
		return nil, err
	}
	return spider, nil
}

// SetFetchInterval set's the spiders wait interval, and calculates the intervalJitter
// value for the random jitter added to the wait. This value is in milliseconds, 1000
// means 1 second.
//
// When fetchInterval > 0:
// time between fetches = fetchInterval + rand(intervalJitter)
// max time between fetches = fetchInterval + intervalJitter
func (s *Spider) SetFetchInterval(i int64) {
	s.fetchInterval = i
	jitter := i / 5 // jitter is, at most 20% of the fetchInterval, Min interval is 10ms
	// 10ms is used as the floor because arbitrary. A larger floor might be reasonable
	if jitter >= 10 {
		s.intervalJitter = jitter
	}
}

// Crawl is the exposed method for starting a crawl at baseURL. The crawl private method
// does the actual work. The depth is the maximum depth, or distance, to crawl from the
// baseURL. If depth == -1, no limits are set and it is expected that the entire site
// will be crawled.
func (s *Spider) Crawl(depth int) (message string, err error) {
	s.maxDepth = depth
	S := Site{URL: s.URL}
	// if we are to respect the robots.txt, set up the info
	if s.RespectRobots {
		s.getRobotsTxt()
	}
	s.Queue.Enqueue(Page{URL: s.URL})
	err = s.crawl(S)
	return fmt.Sprintf("%d nodes were processed; %d external links linking to %d external hosts were not processed", len(s.Pages), len(s.externalLinks), len(s.externalHosts)), err
}

// This crawl does all the work.
func (s *Spider) crawl(fetcher Fetcher) error {
	var err error
	for !s.Queue.IsEmpty() {
		// get next item from queue
		page := s.Queue.Dequeue().(Page)
		// if a depth value was passed and the distance is > depth, we are done
		// depth of 0 means no limit
		if s.maxDepth != -1 && page.distance > s.maxDepth {
			return nil
		}
		// see if this is an external url
		if s.externalURL(page.URL) {
			if s.CheckExternalLinks {
				s.fetchExternalLink(page.URL)
			}
			continue
		}
		// check to see if this url should be skipped for other reasons
		if s.skip(page.URL) {
			// see if the skipped is external and process accordingly
			continue
		}
		s.foundURLs[page.URL.String()] = struct{}{}
		// get the url
		page.body, page.links, err = fetcher.Fetch(page.URL.String())
		// add the page and status to the map. map isn't checked for membership becuase we don't
		// fetch found urls.
		s.Lock()
		s.Pages[page.URL.String()] = page
		s.fetchedURLs[page.URL.String()] = err
		s.Unlock()
		// add the urls that the node contains to the queue
		for _, l := range page.links {
			u, _ := url.Parse(l)
			s.Queue.Enqueue(Page{URL: u, distance: page.distance + 1})
		}
		// if their is a wait between fetches, sleep for that + random jitter
		if s.fetchInterval > 0 {
			wait := s.fetchInterval
			// if there is a value for jitter, add a random jitter
			if s.intervalJitter > 0 {
				wait += rand.Int63n(s.intervalJitter)
			}
			time.Sleep(time.Duration(wait) * time.Millisecond)
		}
	}
	return nil
}

// skip determines whether the url should be skipped.
//   * skip urls that have already been fetched
//   * skip urls that are outside of the basePath
//   * conditionally skip urls that are outside of the current scheme, even if
//     they are within the current pasePath
//   * skip if not allowed by robots
func (s *Spider) skip(u *url.URL) bool {
	s.Lock()
	_, ok := s.foundURLs[u.String()]
	s.Unlock()
	if ok { // if it was found, skip it
		s.addSkippedURL(u)
		return true
	}

	// skip if we are restricted to current scheme
	if s.RestrictToScheme {
		if u.Scheme != s.URL.Scheme {
			s.addSkippedURL(u)
			return true
		}
	}
	if s.RespectRobots {
		ok := s.robotsAllowed(u)
		if !ok {
			return false
		}
	}
	// skip if the url is outside of base
	// remove the scheme + schemePrefix so just the rest of the url is being compared
	if !strings.HasPrefix(u.Path, s.URL.Path) {
		s.addSkippedURL(u)
		return true
	}
	return false
}

// addSkippedURL add's the url info too the skipped info
func (s *Spider) addSkippedURL(u *url.URL) {
	s.Lock()
	s.skippedURLs[u.Path] = struct{}{}
	s.Unlock()
}

// externalURL check's to see if the url is external to the site, host isn't the same.
// and add's that info to the ext structs.
func (s *Spider) externalURL(u *url.URL) bool {
	if u.Host != s.URL.Host {
		s.Lock()
		// see if the host is already in the map
		_, ok := s.externalHosts[u.Host]
		if !ok {
			s.externalHosts[u.Host] = struct{}{}
		}
		// same with url
		_, ok = s.externalLinks[u.String()]
		if !ok {
			s.externalLinks[u.String()] = responseInfo{}
		}
		s.Unlock()
		return true
	}
	return false
}

// fetchExternalLink: fetches an external link and check's it status. Note, this
// does not implement fetcher
func (s *Spider) fetchExternalLink(u *url.URL) error {
	// if this has already benn fetched, don't
	var ri responseInfo
	s.Lock()
	r, _ := s.externalLinks[u.String()]
	if r != ri { // if !0 value, it's been retrieved
		s.Unlock()
		return nil
	}
	s.Unlock()
	resp, err := http.Get(u.String())
	resp.Body.Close()
	if err != nil {
		r.Err = err
		s.Lock()
		s.externalLinks[u.String()] = r
		s.Unlock()
		return err
	}
	r.Status = resp.Status
	r.StatusCode = resp.StatusCode
	s.Lock()
	s.externalLinks[u.String()] = r
	s.Unlock()
	return nil
}

// getRobotsTxt retrieves and processes the site's robot.txt. If the robots.txt doesn't
// exist, it is assumed that everything is allowed.
func (s *Spider) getRobotsTxt() error {
	resp, err := http.Get("http://" + s.URL.Host)
	if err != nil {
		return err
	}
	robots, err := robotstxt.FromResponse(resp)
	if err != nil {
		return err
	}
	s.robots = robots.FindGroup(s.bot)
	return nil
}

// robotsAllowed checks to see if the passed path is allowed by Robots.txt. If the
// robots isn't set, it's always true
func (s *Spider) robotsAllowed(u *url.URL) bool {
	if s.robots != nil {
		return s.robots.Test(u.Path)
	}
	return true
}

// getTokens returns all tokens in the body
func getTokens(body io.Reader) []html.Token {
	tokens := make([]html.Token, 0)
	page := html.NewTokenizer(body)
	for {
		typ := page.Next()
		if typ == html.ErrorToken {
			return tokens
		}
		tokens = append(tokens, page.Token())
	}
}

func init() {
	// We just use math/rand because it's good enough for our purpose.
	rand.Seed(time.Now().UTC().UnixNano())
}
<commit_msg>simplify first unlock() handling in fetchExternalLink<commit_after>// geomi, 거미, is a basic spider for crawling specific webs. It was designed to crawl
// either a site, or a subset of a site, indexing the pages and where they link to.
// Any links that go outside of the provided base URL are noted but not followed.
//
//
// Even though geomi was designed with the intended use being crawling ones own site,
// or a client's site, it does come with some basic behavior configuration to make it
// a friendly bot:
//   * respects ROBOTS.txt TODO
//   * configurable concurrent walkers TODO
//   * configurable wait interval range TODO
//   * configurable max requests per: TODO
//     * min
//     * hour
//     * day
//     * month
//
// To start, go get the geomi package:
//    go get github.com/mohae/geomi
//
// Import it into your code:
//    import github.com/mohae/geomi
//
// Set up the site for the spider:
//   s := geomi.NewSite("http://example.com")
//
// The url that is passed to the NewSite() function is the base case, after which
// a BFS traversal is done.
//
package geomi

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/mohae/utilitybelt/queue"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// Fetcher is an interface that makes it easier to test. In the future, it may be
// useful for other purposes, but that would require exporting the method that uses
// it first.
type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page
	Fetch(url string) (body string, urls []string, err error)
}

//}
// fetched tracks URLs that have been (or are being) fetched.
// The lock must be held while reading from or writing to the map.
var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// a page is a url. This is usually some content wiht a number of elements, but it
// could be something as simple as the url for cat.jpg on your site and represent that
// image.
type Page struct {
	*url.URL
	distance int
	body     string
	links    []string // immediate children
}

// responseInfo contains the status and error information from a get
type responseInfo struct {
	Status     string
	StatusCode int
	Err        error
}

// Site is a type that implements fetcher
type Site struct {
	*url.URL
}

// Implements fetcher.
// TODO: make the design cleaner
func (s Site) Fetch(url string) (body string, urls []string, err error) {
	// see if the passed url is outside of the baseURL
	resp, err := http.Get(url)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()
	buff := &bytes.Buffer{}
	tee := io.TeeReader(resp.Body, buff)
	tokens := getTokens(tee)
	if len(tokens) == 0 {
		return "", nil, fmt.Errorf("%s: nothing in body", url)
	}
	urls, err = s.linksFromTokens(tokens)
	if err != nil {
		return "", nil, err
	}
	return buff.String(), urls, nil
}

// linksFromTokens returns a list of links (href a) found in the token slice
// TODO should internal links be tracked separatly? i.e. record them in a
// separate var (so they don't get fetched)
func (s *Site) linksFromTokens(tokens []html.Token) ([]string, error) {
	var links []string
	for _, token := range tokens {
		if token.Type == html.StartTagToken && token.DataAtom.String() == "a" {
			for _, attr := range token.Attr {
				// We only care about links that aren't #
				if attr.Key == "href" && !strings.HasPrefix(attr.Val, "#") {
					// parse the url
					url, err := url.Parse(attr.Val)
					if err != nil {
						return nil, err
					}
					link := s.URL.ResolveReference(url)
					links = append(links, link.String())
				}
			}
		}
	}
	return links, nil
}

// Spider crawls the target. It contains all information needed to manage the crawl
// including keeping track of work to do, work done, and the results.
type Spider struct {
	*queue.Queue
	sync.Mutex
	wg                 sync.WaitGroup
	*url.URL                  // the start url
	bot                string // the name of the bot for robots
	concurrency        int    // concurrency level of crawling
	fetchInterval      int64  // if > 0, interval, in milliseconds to wait between gets
	intervalJitter     int64  // the max jitter to be added, per fetch. Jitter is a rand with this as max.
	RestrictToScheme   bool   // if true, only crawls urls with the same scheme as baseURL
	RespectRobots      bool   // whether or not to respect the robots.txt
	CheckExternalLinks bool   // Check the status of any links that go to other domains
	robots             *robotstxt.Group
	maxDepth           int
	Pages              map[string]Page
	foundURLs          map[string]struct{}     // keeps track of urls found to prevent recrawling
	fetchedURLs        map[string]error        // urls that have been fetched with their status
	skippedURLs        map[string]struct{}     // urls within the same domain that are not retrieved
	externalHosts      map[string]struct{}     // list of external hosts TODO: elide?
	externalLinks      map[string]responseInfo // list of external links; if fetched,
}

// returns a Spider with the its site's baseUrl set. The baseUrl is the start point for
// the crawl. It is also the restriction on the crawl:
func NewSpider(start string) (*Spider, error) {
	if start == "" {
		return nil, errors.New("newSpider: the start url cannot be empty")
	}
	var err error
	spider := &Spider{Queue: queue.New(128, 0),
		bot:           "geomi",
		RespectRobots: true,
		Pages:         make(map[string]Page),
		foundURLs:     make(map[string]struct{}),
		fetchedURLs:   make(map[string]error),
		skippedURLs:   make(map[string]struct{}),
		externalHosts: make(map[string]struct{}),
		externalLinks: make(map[string]responseInfo),
	}
	spider.URL, err = url.Parse(start)
	if err != nil {
		return nil, err
	}
	return spider, nil
}

// SetFetchInterval set's the spiders wait interval, and calculates the intervalJitter
// value for the random jitter added to the wait. This value is in milliseconds, 1000
// means 1 second.
//
// When fetchInterval > 0:
// time between fetches = fetchInterval + rand(intervalJitter)
// max time between fetches = fetchInterval + intervalJitter
func (s *Spider) SetFetchInterval(i int64) {
	s.fetchInterval = i
	jitter := i / 5 // jitter is, at most 20% of the fetchInterval, Min interval is 10ms
	// 10ms is used as the floor because arbitrary. A larger floor might be reasonable
	if jitter >= 10 {
		s.intervalJitter = jitter
	}
}

// Crawl is the exposed method for starting a crawl at baseURL. The crawl private method
// does the actual work. The depth is the maximum depth, or distance, to crawl from the
// baseURL. If depth == -1, no limits are set and it is expected that the entire site
// will be crawled.
func (s *Spider) Crawl(depth int) (message string, err error) {
	s.maxDepth = depth
	S := Site{URL: s.URL}
	// if we are to respect the robots.txt, set up the info
	if s.RespectRobots {
		s.getRobotsTxt()
	}
	s.Queue.Enqueue(Page{URL: s.URL})
	err = s.crawl(S)
	return fmt.Sprintf("%d nodes were processed; %d external links linking to %d external hosts were not processed", len(s.Pages), len(s.externalLinks), len(s.externalHosts)), err
}

// This crawl does all the work.
func (s *Spider) crawl(fetcher Fetcher) error {
	var err error
	for !s.Queue.IsEmpty() {
		// get next item from queue
		page := s.Queue.Dequeue().(Page)
		// if a depth value was passed and the distance is > depth, we are done
		// depth of 0 means no limit
		if s.maxDepth != -1 && page.distance > s.maxDepth {
			return nil
		}
		// see if this is an external url
		if s.externalURL(page.URL) {
			if s.CheckExternalLinks {
				s.fetchExternalLink(page.URL)
			}
			continue
		}
		// check to see if this url should be skipped for other reasons
		if s.skip(page.URL) {
			// see if the skipped is external and process accordingly
			continue
		}
		s.foundURLs[page.URL.String()] = struct{}{}
		// get the url
		page.body, page.links, err = fetcher.Fetch(page.URL.String())
		// add the page and status to the map. map isn't checked for membership becuase we don't
		// fetch found urls.
		s.Lock()
		s.Pages[page.URL.String()] = page
		s.fetchedURLs[page.URL.String()] = err
		s.Unlock()
		// add the urls that the node contains to the queue
		for _, l := range page.links {
			u, _ := url.Parse(l)
			s.Queue.Enqueue(Page{URL: u, distance: page.distance + 1})
		}
		// if their is a wait between fetches, sleep for that + random jitter
		if s.fetchInterval > 0 {
			wait := s.fetchInterval
			// if there is a value for jitter, add a random jitter
			if s.intervalJitter > 0 {
				wait += rand.Int63n(s.intervalJitter)
			}
			time.Sleep(time.Duration(wait) * time.Millisecond)
		}
	}
	return nil
}

// skip determines whether the url should be skipped.
//   * skip urls that have already been fetched
//   * skip urls that are outside of the basePath
//   * conditionally skip urls that are outside of the current scheme, even if
//     they are within the current pasePath
//   * skip if not allowed by robots
func (s *Spider) skip(u *url.URL) bool {
	s.Lock()
	_, ok := s.foundURLs[u.String()]
	s.Unlock()
	if ok { // if it was found, skip it
		s.addSkippedURL(u)
		return true
	}

	// skip if we are restricted to current scheme
	if s.RestrictToScheme {
		if u.Scheme != s.URL.Scheme {
			s.addSkippedURL(u)
			return true
		}
	}
	if s.RespectRobots {
		ok := s.robotsAllowed(u)
		if !ok {
			return false
		}
	}
	// skip if the url is outside of base
	// remove the scheme + schemePrefix so just the rest of the url is being compared
	if !strings.HasPrefix(u.Path, s.URL.Path) {
		s.addSkippedURL(u)
		return true
	}
	return false
}

// addSkippedURL add's the url info too the skipped info
func (s *Spider) addSkippedURL(u *url.URL) {
	s.Lock()
	s.skippedURLs[u.Path] = struct{}{}
	s.Unlock()
}

// externalURL check's to see if the url is external to the site, host isn't the same.
// and add's that info to the ext structs.
func (s *Spider) externalURL(u *url.URL) bool {
	if u.Host != s.URL.Host {
		s.Lock()
		// see if the host is already in the map
		_, ok := s.externalHosts[u.Host]
		if !ok {
			s.externalHosts[u.Host] = struct{}{}
		}
		// same with url
		_, ok = s.externalLinks[u.String()]
		if !ok {
			s.externalLinks[u.String()] = responseInfo{}
		}
		s.Unlock()
		return true
	}
	return false
}

// fetchExternalLink: fetches an external link and check's it status. Note, this
// does not implement fetcher
func (s *Spider) fetchExternalLink(u *url.URL) error {
	// if this has already benn fetched, don't
	var ri responseInfo
	s.Lock()
	r, _ := s.externalLinks[u.String()]
	s.Unlock()
	if r != ri { // if !0 value, it's been retrieved

		return nil
	}
	s.Unlock()
	resp, err := http.Get(u.String())
	resp.Body.Close()
	if err != nil {
		r.Err = err
		s.Lock()
		s.externalLinks[u.String()] = r
		s.Unlock()
		return err
	}
	r.Status = resp.Status
	r.StatusCode = resp.StatusCode
	s.Lock()
	s.externalLinks[u.String()] = r
	s.Unlock()
	return nil
}

// getRobotsTxt retrieves and processes the site's robot.txt. If the robots.txt doesn't
// exist, it is assumed that everything is allowed.
func (s *Spider) getRobotsTxt() error {
	resp, err := http.Get("http://" + s.URL.Host)
	if err != nil {
		return err
	}
	robots, err := robotstxt.FromResponse(resp)
	if err != nil {
		return err
	}
	s.robots = robots.FindGroup(s.bot)
	return nil
}

// robotsAllowed checks to see if the passed path is allowed by Robots.txt. If the
// robots isn't set, it's always true
func (s *Spider) robotsAllowed(u *url.URL) bool {
	if s.robots != nil {
		return s.robots.Test(u.Path)
	}
	return true
}

// getTokens returns all tokens in the body
func getTokens(body io.Reader) []html.Token {
	tokens := make([]html.Token, 0)
	page := html.NewTokenizer(body)
	for {
		typ := page.Next()
		if typ == html.ErrorToken {
			return tokens
		}
		tokens = append(tokens, page.Token())
	}
}

func init() {
	// We just use math/rand because it's good enough for our purpose.
	rand.Seed(time.Now().UTC().UnixNano())
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "strings"
  "flag"
  "os"
  // "io/ioutil"
  // "net/http"

)

// this curler is currently purpose-built to hit neo4j hosts and figure out
// if the host is master/slave, or up at all.
// I'll expand it's utility just a bit later.
// Also, this is ONLY functional with HA neo4j, not causal clustering or
// singletons at the moment. I'll fix that later. Right now I'm all POC all the time
// like, you know, POC to prod without any changes ftw

func main() {
  rawHostAddress := flag.String("host", "", "enter a host address or IP") // using fqdn relies on faithful DNS resolution
  rawNeo4jRole := flag.String("role", "master", "enter master or slave") // Neo4j HA cluster has 3 roles: master, slave, arbiter. We won't test for the latter
  rawFlagAuthContent := flag.String("auth", "", "enter JUST the base64-encoded auth content. We will add the rest for you. Example: aaAaaaAAaaaAbbbBb678bV==")

  flag.Parse() // parse those flags!

  hostAddress := strings.ToLower(*rawHostAddress)
  neo4jRole := strings.ToLower(*rawNeo4jRole)
  flagAuthContent := *rawFlagAuthContent
  fmt.Println("debug: printing hostAddress", hostAddress)
  fmt.Println("debug: printing neo4jRole", neo4jRole)
  fmt.Println("debug: printing flagAuthContent", flagAuthContent)
  // make sure neo4jRole is master or slave
  if neo4jRole != "master" && neo4jRole != "slave" {
    fmt.Println("Please ONLY use master or slave with the -role flag")
    os.Exit(1)
  }
  // rawHostAddress/hostAddress cannot be empty
  if hostAddress == "" {
    fmt.Println("Enter an address for your curl target")
    os.Exit(1)
  }
  // auth content cannot be empty in this specific case
  if flagAuthContent == "" {
    fmt.Println("You must enter a base64-encoded auth string")
    os.Exit(1)
  }

  curlTarget := fmt.Sprintf("http://%v:7474/db/manage/server/ha/%v", hostAddress, neo4jRole)

  fmt.Println(curlTarget)

  // create the curl
  req, err := http.NewRequest("GET", curlTarget, nil)
  if err != nil {
    os.Exit(1)
  }
  // add in authorization
  req.Header.Set("Authorization", flagAuthContent)
  // give it a method to ingest the response
  res, _ := http.DefaultClient.Do(req)
  // don't forget to eventually close the session
  defer res.Body.Close()
  // read the body into 'body'
  rawBody, _ := ioutil.ReadAll(res.Body)
  body := strings.ToLower(*rawBody)

  fmt.Println("debug: printing contents of 'body' variable", body)

  if body != neo4jRole {
    fmt.Println("Warning: Neo4j Server currently set to wrong role. Role should be", neo4jRole)
    os.Exit(1)
  }
}
<commit_msg>fixed dumb mistakes, uncommented imports I'm using now<commit_after>package main

import (
  "fmt"
  "strings"
  "flag"
  "os"
  "io/ioutil"
  "net/http"

)

// this curler is currently purpose-built to hit neo4j hosts and figure out
// if the host is master/slave, or up at all.
// I'll expand it's utility just a bit later.
// Also, this is ONLY functional with HA neo4j, not causal clustering or
// singletons at the moment. I'll fix that later. Right now I'm all POC all the time
// like, you know, POC to prod without any changes ftw

func main() {
  rawHostAddress := flag.String("host", "", "enter a host address or IP") // using fqdn relies on faithful DNS resolution
  rawNeo4jRole := flag.String("role", "master", "enter master or slave") // Neo4j HA cluster has 3 roles: master, slave, arbiter. We won't test for the latter
  rawFlagAuthContent := flag.String("auth", "", "enter JUST the base64-encoded auth content. We will add the rest for you. Example: aaAaaaAAaaaAbbbBb678bV==")

  flag.Parse() // parse those flags!

  hostAddress := strings.ToLower(*rawHostAddress)
  neo4jRole := strings.ToLower(*rawNeo4jRole)
  flagAuthContent := *rawFlagAuthContent
  fmt.Println("debug: printing hostAddress", hostAddress)
  fmt.Println("debug: printing neo4jRole", neo4jRole)
  fmt.Println("debug: printing flagAuthContent", flagAuthContent)
  // make sure neo4jRole is master or slave
  if neo4jRole != "master" && neo4jRole != "slave" {
    fmt.Println("Please ONLY use master or slave with the -role flag")
    os.Exit(1)
  }
  // rawHostAddress/hostAddress cannot be empty
  if hostAddress == "" {
    fmt.Println("Enter an address for your curl target")
    os.Exit(1)
  }
  // auth content cannot be empty in this specific case
  if flagAuthContent == "" {
    fmt.Println("You must enter a base64-encoded auth string")
    os.Exit(1)
  }

  curlTarget := fmt.Sprintf("http://%v:7474/db/manage/server/ha/%v", hostAddress, neo4jRole)

  fmt.Println(curlTarget)

  // create the curl
  req, err := http.NewRequest("GET", curlTarget, nil)
  if err != nil {
    os.Exit(1)
  }
  // add in authorization
  req.Header.Set("Authorization", flagAuthContent)
  // give it a method to ingest the response
  res, _ := http.DefaultClient.Do(req)
  // don't forget to eventually close the session
  defer res.Body.Close()
  // read the body into 'body'
  rawBody, _ := ioutil.ReadAll(res.Body)
  body := strings.ToLower(string(rawBody))

  fmt.Println("debug: printing contents of 'body' variable", body)

  if body != neo4jRole {
    fmt.Println("Warning: Neo4j Server currently set to wrong role. Role should be", neo4jRole)
    os.Exit(1)
  }
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
	"log"
	"net/http"
	"time"
)

// type Annotation struct is defined in annotation.go

func addAnnotationHandler(rw http.ResponseWriter, req *http.Request) {
	log.Printf("\n\naddAnnotation-request: %v\n", req)

	// get session
	cs, err := getClientSession(req.Header.Get(headerKeySessionKey))
	if err != nil {
		http.Error(rw, "error", http.StatusInternalServerError)
		return
	}
	defer cs.done()

	// get account
	acc := cs.account
	if acc == nil {
		http.Error(rw, "forbidden", http.StatusForbidden)
		return
	}

	switch req.Method {
	case "POST":
		body, _ := ioutil.ReadAll(req.Body)
		log.Printf("\n\nbody is %s\n", string(body))
		annot := &Annotation{}
		err := json.Unmarshal(body, annot)
		if err != nil {
			log.Printf("\n\nJSON unmarshal error %#v\n", err)
			http.Error(rw, "JSON unmarshal error", http.StatusBadRequest) // 400
			return
		}

		// Set every other field to things we control.
		annot.ID = bson.NewObjectId()
		annot.Annotator = acc.Username
		annot.CreateDate = time.Now()
		annot.Comments = []Comment{}

		log.Printf("\n\nAnnotation to insert is: %#v\n", *annot)

		err = insertAnnotation(annot)
		if err != nil {
			log.Printf("Error inserting annotation: error %#v\n", err)
			http.Error(rw, "error inserting annotation", http.StatusInternalServerError) // 500
			return
		}

		rw.WriteHeader(200)
		rw.Write([]byte(`OK, inserted`))
		return
	default:
		http.Error(rw, "error", http.StatusMethodNotAllowed) // 405
	}
}

<commit_msg>gofmt...<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
	"log"
	"net/http"
	"time"
)

// type Annotation struct is defined in annotation.go

func addAnnotationHandler(rw http.ResponseWriter, req *http.Request) {
	log.Printf("\n\naddAnnotation-request: %v\n", req)

	// get session
	cs, err := getClientSession(req.Header.Get(headerKeySessionKey))
	if err != nil {
		http.Error(rw, "error", http.StatusInternalServerError)
		return
	}
	defer cs.done()

	// get account
	acc := cs.account
	if acc == nil {
		http.Error(rw, "forbidden", http.StatusForbidden)
		return
	}

	switch req.Method {
	case "POST":
		body, _ := ioutil.ReadAll(req.Body)
		log.Printf("\n\nbody is %s\n", string(body))
		annot := &Annotation{}
		err := json.Unmarshal(body, annot)
		if err != nil {
			log.Printf("\n\nJSON unmarshal error %#v\n", err)
			http.Error(rw, "JSON unmarshal error", http.StatusBadRequest) // 400
			return
		}

		// Set every other field to things we control.
		annot.ID = bson.NewObjectId()
		annot.Annotator = acc.Username
		annot.CreateDate = time.Now()
		annot.Comments = []Comment{}

		log.Printf("\n\nAnnotation to insert is: %#v\n", *annot)

		err = insertAnnotation(annot)
		if err != nil {
			log.Printf("Error inserting annotation: error %#v\n", err)
			http.Error(rw, "error inserting annotation", http.StatusInternalServerError) // 500
			return
		}

		rw.WriteHeader(200)
		rw.Write([]byte(`OK, inserted`))
		return
	default:
		http.Error(rw, "error", http.StatusMethodNotAllowed) // 405
	}
}
<|endoftext|>


<commit_before><commit_msg>Fix orientation default value<commit_after><|endoftext|>


<commit_before>package followingfeed

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/tools/logger"
	"socialapi/models"

	verbalexpressions "github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/jinzhu/gorm"
)

// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
// will find [foo, bar, 123]
// will not find [' baz', '-`3sdf']
// here is the regex -> (?m)(?:#)(\w+)(?: )
var topicRegex = verbalexpressions.New().
	Find("#").
	BeginCapture().
	Word().
	EndCapture().
	Find(" ").
	Regex()

type Action func(*TopicFeedController, *models.ChannelMessage) error

type TopicFeedController struct {
	routes map[string]Action
	log    logger.Log
}

var HandlerNotFoundErr = errors.New("Handler Not Found")

func NewTopicFeedController(log logger.Log) *TopicFeedController {
	// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
	// res := topicRegex.FindAllStringSubmatch(s, -1)
	// fmt.Println("res", res)
	// fmt.Println("res", res[0][1])
	// fmt.Println("res", res[1][1])
	// fmt.Println(topicRegex.String())

	ffc := &TopicFeedController{
		log: log,
	}

	routes := map[string]Action{
		"channel_message_created": (*TopicFeedController).MessageSaved,
		"channel_message_update":  (*TopicFeedController).MessageUpdated,
		"channel_message_deleted": (*TopicFeedController).MessageDeleted,
	}

	ffc.routes = routes

	return ffc
}

func (f *TopicFeedController) HandleEvent(event string, data []byte) error {
	f.log.Debug("New Event Recieved %s", event)
	handler, ok := f.routes[event]
	if !ok {
		return HandlerNotFoundErr
	}

	cm, err := mapMessage(data)
	if err != nil {
		return err
	}

	res, err := isEligible(cm)
	if err != nil {
		return err
	}

	if !res {
		return nil
	}

	return handler(f, cm)
}

func (f *TopicFeedController) MessageSaved(data *models.ChannelMessage) error {

	res := topicRegex.FindAllStringSubmatch(data.Body, -1)
	if len(res) == 0 {
		f.log.Debug("Message doesnt have any topic Body: %s", data.Body)
		return nil
	}

	c, err := fetchChannel(data.InitialChannelId)
	if err != nil {
		return err
	}

	topics := map[string]struct{}{}

	for _, ele := range res {
		topics[ele[1]] = struct{}{}
	}

	for topic := range topics {
		channelName := topic
		tc, err := fetchTopicChannel(c.Group, channelName)
		if err != nil && err != gorm.RecordNotFound {
			return err
		}

		if err == gorm.RecordNotFound {
			tc, err = createTopicChannel(data.AccountId, c.Group, channelName, c.Privacy)
			if err != nil {
				return err
			}
		}

		_, err = tc.AddMessage(data.Id)
		if err != nil {
			return err
		}

	}

	return nil
}

func extractTopics(body string) []string {
	flattened := make([]string, 0)

	res := topicRegex.FindAllStringSubmatch(body, -1)
	if len(res) == 0 {
		return flattened
	}

	topics := map[string]struct{}{}
	// remove duplicate tag usages
	for _, ele := range res {
		topics[ele[1]] = struct{}{}
	}

	for topic := range topics {
		flattened = append(flattened, topic)
	}

	return flattened
}

func (f *TopicFeedController) MessageUpdated(data *models.ChannelMessage) error {
	f.log.Debug("udpate message %s", data.Id)
	// fetch message's current topics from the db
	channels, err := fetchMessageChannels(data.Id)
	if err != nil {
		return err
	}

	// get current topics from
	topics := extractTopics(data.Body)
	if topics == nil {
		return nil
	}

	// if message and the topics dont have any item, we can safely return
	if len(channels) == 0 && len(topics) == 0 {
		return nil
	}

	res := getTopicDiff(channels, topics)

	// add messages
	if len(res["added"]) > 0 {
		initialChannel, err := fetchChannel(data.InitialChannelId)
		if err != nil {
			return err
		}

		if err := ensureChannelMessages(initialChannel, data, res["added"]); err != nil {
			return err
		}
	}

	// delete messages
	if len(res["deleted"]) > 0 {
		if err := deleteChannelMessages(channels, data, res["deleted"]); err != nil {
			return err
		}
	}

	return nil
}

func deleteChannelMessages(channels []models.Channel, data *models.ChannelMessage, toBeDeletedTopics []string) error {
	for _, channel := range channels {
		for _, topic := range toBeDeletedTopics {
			if channel.Name != topic {
				continue
			}

			cml := models.NewChannelMessageList()
			selector := map[string]interface{}{
				"mesage_id":  data.Id,
				"channel_id": channel.Id,
			}

			if err := cml.DeleteMessagesBySelector(selector); err != nil {
				return err
			}
		}
	}
	return nil
}
func getTopicDiff(channels []models.Channel, topics []string) map[string][]string {
	res := make(map[string][]string)

	// aggregate all channel names into map
	channelNames := map[string]struct{}{}
	for _, channel := range channels {
		channelNames[channel.Name] = struct{}{}
	}

	// range over new topics, bacause we are gonna remove
	// unused channels
	for _, topic := range topics {
		found := false
		for channelName := range channelNames {
			if channelName == topic {
				found = true
			}
		}
		if !found {
			res["added"] = append(res["added"], topic)
		} else {
			// if we have topic in channels
			// do remove it because at the end we are gonna mark
			// channels as deleted which are still in channelNames
			delete(channelNames, topic)
		}
	}
	// flatten the deleted channel names
	for channelName := range channelNames {
		res["deleted"] = append(res["deleted"], channelName)
	}

	return res
}

func (f *TopicFeedController) MessageDeleted(data *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	selector := map[string]interface{}{
		"mesage_id": data.Id,
	}

	if err := cml.DeleteMessagesBySelector(selector); err != nil {
		return err
	}
	return nil
}

func mapMessage(data []byte) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	if err := json.Unmarshal(data, cm); err != nil {
		return nil, err
	}

	return cm, nil
}

func isEligible(cm *models.ChannelMessage) (bool, error) {
	if cm.InitialChannelId == 0 {
		return false, nil
	}

	if cm.Type != models.ChannelMessage_TYPE_POST {
		return false, nil
	}

	return true, nil
}

// todo add caching here
func fetchChannel(channelId int64) (*models.Channel, error) {
	c := models.NewChannel()
	c.Id = channelId
	// todo - fetch only name here
	if err := c.Fetch(); err != nil {
		return nil, err
	}

	return c, nil
}

// todo add caching here
func fetchTopicChannel(groupName, channelName string) (*models.Channel, error) {
	c := models.NewChannel()

	selector := map[string]interface{}{
		"group": groupName,
		"name":  channelName,
		"type":  models.Channel_TYPE_TOPIC,
	}

	err := c.One(selector)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func createTopicChannel(creatorId int64, groupName, channelName, privacy string) (*models.Channel, error) {
	c := models.NewChannel()
	c.Name = channelName
	c.CreatorId = creatorId
	c.Group = groupName
	c.Purpose = fmt.Sprintf("Channel for %s topic", channelName)
	c.Type = models.Channel_TYPE_TOPIC
	c.Privacy = privacy
	if err := c.Create(); err != nil {
		return nil, err
	}

	return c, nil
}
<commit_msg>social: add a function for fetching channels of messages<commit_after>package followingfeed

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/tools/logger"
	"socialapi/models"

	verbalexpressions "github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/jinzhu/gorm"
)

// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
// will find [foo, bar, 123]
// will not find [' baz', '-`3sdf']
// here is the regex -> (?m)(?:#)(\w+)(?: )
var topicRegex = verbalexpressions.New().
	Find("#").
	BeginCapture().
	Word().
	EndCapture().
	Find(" ").
	Regex()

type Action func(*TopicFeedController, *models.ChannelMessage) error

type TopicFeedController struct {
	routes map[string]Action
	log    logger.Log
}

var HandlerNotFoundErr = errors.New("Handler Not Found")

func NewTopicFeedController(log logger.Log) *TopicFeedController {
	// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
	// res := topicRegex.FindAllStringSubmatch(s, -1)
	// fmt.Println("res", res)
	// fmt.Println("res", res[0][1])
	// fmt.Println("res", res[1][1])
	// fmt.Println(topicRegex.String())

	ffc := &TopicFeedController{
		log: log,
	}

	routes := map[string]Action{
		"channel_message_created": (*TopicFeedController).MessageSaved,
		"channel_message_update":  (*TopicFeedController).MessageUpdated,
		"channel_message_deleted": (*TopicFeedController).MessageDeleted,
	}

	ffc.routes = routes

	return ffc
}

func (f *TopicFeedController) HandleEvent(event string, data []byte) error {
	f.log.Debug("New Event Recieved %s", event)
	handler, ok := f.routes[event]
	if !ok {
		return HandlerNotFoundErr
	}

	cm, err := mapMessage(data)
	if err != nil {
		return err
	}

	res, err := isEligible(cm)
	if err != nil {
		return err
	}

	if !res {
		return nil
	}

	return handler(f, cm)
}

func (f *TopicFeedController) MessageSaved(data *models.ChannelMessage) error {

	res := topicRegex.FindAllStringSubmatch(data.Body, -1)
	if len(res) == 0 {
		f.log.Debug("Message doesnt have any topic Body: %s", data.Body)
		return nil
	}

	c, err := fetchChannel(data.InitialChannelId)
	if err != nil {
		return err
	}

	topics := map[string]struct{}{}

	for _, ele := range res {
		topics[ele[1]] = struct{}{}
	}

	for topic := range topics {
		channelName := topic
		tc, err := fetchTopicChannel(c.Group, channelName)
		if err != nil && err != gorm.RecordNotFound {
			return err
		}

		if err == gorm.RecordNotFound {
			tc, err = createTopicChannel(data.AccountId, c.Group, channelName, c.Privacy)
			if err != nil {
				return err
			}
		}

		_, err = tc.AddMessage(data.Id)
		if err != nil {
			return err
		}

	}

	return nil
}

func extractTopics(body string) []string {
	flattened := make([]string, 0)

	res := topicRegex.FindAllStringSubmatch(body, -1)
	if len(res) == 0 {
		return flattened
	}

	topics := map[string]struct{}{}
	// remove duplicate tag usages
	for _, ele := range res {
		topics[ele[1]] = struct{}{}
	}

	for topic := range topics {
		flattened = append(flattened, topic)
	}

	return flattened
}

func (f *TopicFeedController) MessageUpdated(data *models.ChannelMessage) error {
	f.log.Debug("udpate message %s", data.Id)
	// fetch message's current topics from the db
	channels, err := fetchMessageChannels(data.Id)
	if err != nil {
		return err
	}

	// get current topics from
	topics := extractTopics(data.Body)
	if topics == nil {
		return nil
	}

	// if message and the topics dont have any item, we can safely return
	if len(channels) == 0 && len(topics) == 0 {
		return nil
	}

	res := getTopicDiff(channels, topics)

	// add messages
	if len(res["added"]) > 0 {
		initialChannel, err := fetchChannel(data.InitialChannelId)
		if err != nil {
			return err
		}

		if err := ensureChannelMessages(initialChannel, data, res["added"]); err != nil {
			return err
		}
	}

	// delete messages
	if len(res["deleted"]) > 0 {
		if err := deleteChannelMessages(channels, data, res["deleted"]); err != nil {
			return err
		}
	}

	return nil
}

func deleteChannelMessages(channels []models.Channel, data *models.ChannelMessage, toBeDeletedTopics []string) error {
	for _, channel := range channels {
		for _, topic := range toBeDeletedTopics {
			if channel.Name != topic {
				continue
			}

			cml := models.NewChannelMessageList()
			selector := map[string]interface{}{
				"mesage_id":  data.Id,
				"channel_id": channel.Id,
			}

			if err := cml.DeleteMessagesBySelector(selector); err != nil {
				return err
			}
		}
	}
	return nil
}

func fetchMessageChannels(messageId int64) ([]models.Channel, error) {
	cml := models.NewChannelMessageList()
	return cml.FetchMessageChannels(messageId)
}

func getTopicDiff(channels []models.Channel, topics []string) map[string][]string {
	res := make(map[string][]string)

	// aggregate all channel names into map
	channelNames := map[string]struct{}{}
	for _, channel := range channels {
		channelNames[channel.Name] = struct{}{}
	}

	// range over new topics, bacause we are gonna remove
	// unused channels
	for _, topic := range topics {
		found := false
		for channelName := range channelNames {
			if channelName == topic {
				found = true
			}
		}
		if !found {
			res["added"] = append(res["added"], topic)
		} else {
			// if we have topic in channels
			// do remove it because at the end we are gonna mark
			// channels as deleted which are still in channelNames
			delete(channelNames, topic)
		}
	}
	// flatten the deleted channel names
	for channelName := range channelNames {
		res["deleted"] = append(res["deleted"], channelName)
	}

	return res
}

func (f *TopicFeedController) MessageDeleted(data *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	selector := map[string]interface{}{
		"mesage_id": data.Id,
	}

	if err := cml.DeleteMessagesBySelector(selector); err != nil {
		return err
	}
	return nil
}

func mapMessage(data []byte) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	if err := json.Unmarshal(data, cm); err != nil {
		return nil, err
	}

	return cm, nil
}

func isEligible(cm *models.ChannelMessage) (bool, error) {
	if cm.InitialChannelId == 0 {
		return false, nil
	}

	if cm.Type != models.ChannelMessage_TYPE_POST {
		return false, nil
	}

	return true, nil
}

// todo add caching here
func fetchChannel(channelId int64) (*models.Channel, error) {
	c := models.NewChannel()
	c.Id = channelId
	// todo - fetch only name here
	if err := c.Fetch(); err != nil {
		return nil, err
	}

	return c, nil
}

// todo add caching here
func fetchTopicChannel(groupName, channelName string) (*models.Channel, error) {
	c := models.NewChannel()

	selector := map[string]interface{}{
		"group": groupName,
		"name":  channelName,
		"type":  models.Channel_TYPE_TOPIC,
	}

	err := c.One(selector)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func createTopicChannel(creatorId int64, groupName, channelName, privacy string) (*models.Channel, error) {
	c := models.NewChannel()
	c.Name = channelName
	c.CreatorId = creatorId
	c.Group = groupName
	c.Purpose = fmt.Sprintf("Channel for %s topic", channelName)
	c.Type = models.Channel_TYPE_TOPIC
	c.Privacy = privacy
	if err := c.Create(); err != nil {
		return nil, err
	}

	return c, nil
}
<|endoftext|>


<commit_before>package toolbox_test

import (
	"github.com/stretchr/testify/assert"
	"github.com/viant/toolbox"
	"testing"
)

func TestNewFileSetInfoInfo(t *testing.T) {

	fileSetInfo, err := toolbox.NewFileSetInfo("./fileset_info_test/")
	if err != nil {
		panic(err)
	}
	assert.Equal(t, 2, len(fileSetInfo.FilesInfo()))

	fileInfo := fileSetInfo.FileInfo("user_test.go")
	assert.NotNil(t, fileInfo)

	addresses := fileSetInfo.Type("Addresses")
	assert.NotNil(t, addresses)

	assert.False(t, fileInfo.HasType("F"))
	assert.True(t, fileInfo.HasType("User"))

	assert.Equal(t, 7, len(fileInfo.Types()))

	address := fileSetInfo.Type("Address")
	assert.NotNil(t, address)

	assert.Equal(t, 2, len(address.Fields()))
	country := address.Field("Country")
	assert.NotNil(t, country)
	assert.True(t, country.IsAnonymous)

	z := fileSetInfo.Type("Z")
	assert.NotNil(t, z)

	address2 := fileSetInfo.Type("Address2")
	assert.Nil(t, address2)

	userInfo := fileInfo.Type("User")
	assert.NotNil(t, userInfo)

	assert.True(t, userInfo.HasField("ID"))
	assert.True(t, userInfo.HasField("Name"))
	assert.False(t, userInfo.HasField("FF"))

	assert.Equal(t, 11, len(userInfo.Fields()))

	idInfo := userInfo.Field("ID")
	assert.True(t, idInfo.IsPointer)
	assert.Equal(t, "int", idInfo.TypeName)
	assert.Equal(t, true, idInfo.IsPointer)

	dobInfo := userInfo.Field("DateOfBirth")

	assert.Equal(t, "time.Time", dobInfo.TypeName)
	assert.Equal(t, "time", dobInfo.TypePackage)

	assert.Equal(t, "`foo=\"bar\"`", dobInfo.Tag)

	addressPointer := userInfo.Field("AddressPointer")
	assert.NotNil(t, addressPointer)
	assert.Equal(t, "Address", addressPointer.TypeName)

	cInfo := userInfo.Field("C")
	assert.True(t, cInfo.IsChannel)

	mInfo := userInfo.Field("M")
	assert.True(t, mInfo.IsMap)
	assert.Equal(t, "string", mInfo.KeyTypeName)
	assert.Equal(t, "[]string", mInfo.ValueTypeName)

	intsInfo := userInfo.Field("Ints")
	assert.True(t, intsInfo.IsSlice)
	assert.Equal(t, "my comments", userInfo.Comment)

	assert.False(t, userInfo.HasReceiver("Abc"))

	assert.Equal(t, 3, len(userInfo.Receivers()))
	assert.True(t, userInfo.HasReceiver("Test"))
	assert.True(t, userInfo.HasReceiver("Test2"))

	receiver := userInfo.Receiver("Test")
	assert.NotNil(t, receiver)

	appointments := userInfo.Field("Appointments")
	assert.NotNil(t, appointments)
	assert.Equal(t, "time.Time", appointments.ComponentType)

}
<commit_msg>updated test<commit_after>package toolbox_test

import (
	"github.com/stretchr/testify/assert"
	"github.com/viant/toolbox"
	"testing"
)

func TestNewFileSetInfoInfo(t *testing.T) {

	fileSetInfo, err := toolbox.NewFileSetInfo("./fileset_info_test/")
	if err != nil {
		panic(err)
	}
	assert.True(t, len(fileSetInfo.FilesInfo()) > 0)

	fileInfo := fileSetInfo.FileInfo("user_test.go")
	assert.NotNil(t, fileInfo)

	addresses := fileSetInfo.Type("Addresses")
	assert.NotNil(t, addresses)

	assert.False(t, fileInfo.HasType("F"))
	assert.True(t, fileInfo.HasType("User"))

	assert.Equal(t, 7, len(fileInfo.Types()))

	address := fileSetInfo.Type("Address")
	assert.NotNil(t, address)

	assert.Equal(t, 2, len(address.Fields()))
	country := address.Field("Country")
	assert.NotNil(t, country)
	assert.True(t, country.IsAnonymous)

	z := fileSetInfo.Type("Z")
	assert.NotNil(t, z)

	address2 := fileSetInfo.Type("Address2")
	assert.Nil(t, address2)

	userInfo := fileInfo.Type("User")
	assert.NotNil(t, userInfo)

	assert.True(t, userInfo.HasField("ID"))
	assert.True(t, userInfo.HasField("Name"))
	assert.False(t, userInfo.HasField("FF"))

	assert.Equal(t, 11, len(userInfo.Fields()))

	idInfo := userInfo.Field("ID")
	assert.True(t, idInfo.IsPointer)
	assert.Equal(t, "int", idInfo.TypeName)
	assert.Equal(t, true, idInfo.IsPointer)

	dobInfo := userInfo.Field("DateOfBirth")

	assert.Equal(t, "time.Time", dobInfo.TypeName)
	assert.Equal(t, "time", dobInfo.TypePackage)

	assert.Equal(t, "`foo=\"bar\"`", dobInfo.Tag)

	addressPointer := userInfo.Field("AddressPointer")
	assert.NotNil(t, addressPointer)
	assert.Equal(t, "Address", addressPointer.TypeName)

	cInfo := userInfo.Field("C")
	assert.True(t, cInfo.IsChannel)

	mInfo := userInfo.Field("M")
	assert.True(t, mInfo.IsMap)
	assert.Equal(t, "string", mInfo.KeyTypeName)
	assert.Equal(t, "[]string", mInfo.ValueTypeName)

	intsInfo := userInfo.Field("Ints")
	assert.True(t, intsInfo.IsSlice)
	assert.Equal(t, "my comments", userInfo.Comment)

	assert.False(t, userInfo.HasReceiver("Abc"))

	assert.True(t, len(userInfo.Receivers()) > 1)
	assert.True(t, userInfo.HasReceiver("Test"))
	assert.True(t, userInfo.HasReceiver("Test2"))

	receiver := userInfo.Receiver("Test")
	assert.NotNil(t, receiver)

	appointments := userInfo.Field("Appointments")
	assert.NotNil(t, appointments)
	assert.Equal(t, "time.Time", appointments.ComponentType)

}
<|endoftext|>